diff --git a/.github/actions/get-job-data/action.yml b/.github/actions/get-job-data/action.yml new file mode 100644 index 0000000000000..a0b15c0dd8030 --- /dev/null +++ b/.github/actions/get-job-data/action.yml @@ -0,0 +1,93 @@ +name: 'Get Job Data from GitHub Actions' +description: 'Fetches the GitHub Actions job data from the GitHub API' +inputs: + job-name: + description: 'The name of the job to find' + required: true + github-token: + description: 'GitHub token for API authentication' + required: true + repository: + description: 'Repository in owner/repo format' + required: true + run-id: + description: 'GitHub Actions run ID' + required: true +outputs: + job_html_url: + description: 'The HTML URL of the job' + value: ${{ steps.get_url.outputs.job_html_url }} +runs: + using: 'composite' + steps: + - name: Fetch job URL from GitHub API + id: get_url + shell: bash + run: | + # Fetch the numeric job ID from GitHub API + CURL_ERROR_FILE=$(mktemp) + NETRC_FILE=$(mktemp) + + # Write GitHub API credentials to a temporary netrc file to avoid + # passing the token directly on the curl command line. + printf '%s\n' \ + 'machine api.github.com' \ + ' login x-access-token' \ + " password ${{ inputs.github-token }}" \ + > "$NETRC_FILE" + chmod 600 "$NETRC_FILE" + + # Ensure temporary file cleanup on exit + cleanup() { + if [ -n "$CURL_ERROR_FILE" ] && [ -f "$CURL_ERROR_FILE" ]; then + rm -f "$CURL_ERROR_FILE" + fi + if [ -n "$NETRC_FILE" ] && [ -f "$NETRC_FILE" ]; then + rm -f "$NETRC_FILE" + fi + } + trap cleanup EXIT + + API_RESPONSE=$(curl -sS -w "\n%{http_code}" \ + --netrc-file "$NETRC_FILE" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/${{ inputs.repository }}/actions/runs/${{ inputs.run-id }}/jobs" 2>"$CURL_ERROR_FILE") + + CURL_EXIT_CODE=$? + if [ $CURL_EXIT_CODE -ne 0 ]; then + echo "❌ ERROR: curl request to GitHub API failed with exit code $CURL_EXIT_CODE" + if [ -s "$CURL_ERROR_FILE" ]; then + echo "curl error output:" + cat "$CURL_ERROR_FILE" + fi + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + HTTP_CODE=$(echo "$API_RESPONSE" | tail -n1) + RESPONSE_BODY=$(echo "$API_RESPONSE" | sed '$d') + + if [ "$HTTP_CODE" != "200" ]; then + echo "⚠️ WARNING: GitHub API request failed with $HTTP_CODE" + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + EXPECTED_JOB_NAME="${{ inputs.job-name }}" + JOB_URL=$(echo "$RESPONSE_BODY" | jq -r \ + --arg job_name "$EXPECTED_JOB_NAME" \ + '.jobs[] | select(.name == $job_name) | .html_url') + + if [ -z "$JOB_URL" ] || [ "$JOB_URL" = "null" ]; then + echo "⚠️ WARNING: Failed to extract job URL from response for job name '$EXPECTED_JOB_NAME'." + echo "Possible causes:" + echo " - The job name does not match exactly (including spaces and case)." + echo " - The job has not started yet at the time this action ran." + echo " - The GitHub API response format was unexpected." + echo "Please verify that the job has started before this action runs and double-check the exact job name in the GitHub Actions UI." + echo "job_html_url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "job_html_url=$JOB_URL" >> "$GITHUB_OUTPUT" + echo "Job URL: $JOB_URL" diff --git a/.github/scripts/detect-app-cache.sh b/.github/scripts/detect-app-cache.sh new file mode 100755 index 0000000000000..0803d27156a63 --- /dev/null +++ b/.github/scripts/detect-app-cache.sh @@ -0,0 +1,377 @@ +#!/bin/bash + +# SPDX-FileCopyrightText: 2025 STRATO AG +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Script to detect which apps need building vs. can be restored from cache +# Supports multiple cache sources: GitHub Actions cache and JFrog Artifactory +# Outputs JSON arrays for apps to build and apps to restore + +set -e # Exit on error +set -u # Exit on undefined variable +set -o pipefail # Exit if any command in pipeline fails + +# Required environment variables +: "${GH_TOKEN:?GH_TOKEN not set}" +: "${CACHE_VERSION:?CACHE_VERSION not set}" +: "${FORCE_REBUILD:?FORCE_REBUILD not set}" +: "${ARTIFACTORY_REPOSITORY_SNAPSHOT:?ARTIFACTORY_REPOSITORY_SNAPSHOT not set}" + +# Optional variables +APPS_TO_REBUILD="${APPS_TO_REBUILD:-}" +JF_URL="${JF_URL:-}" +JF_USER="${JF_USER:-}" +JF_ACCESS_TOKEN="${JF_ACCESS_TOKEN:-}" + +# Input: MATRIX (JSON array of app configurations) +# Input: GITHUB_REF (current GitHub ref) +# Input: GITHUB_STEP_SUMMARY (path to step summary file) + +# Outputs to $GITHUB_OUTPUT: +# - apps_to_build: JSON array of apps that need building +# - apps_to_restore: JSON array of apps that can be restored from cache +# - apps_sha_map: JSON object mapping app names to their SHAs +# - has_apps_to_build: boolean flag +# - has_apps_to_restore: boolean flag + +echo "Collecting app SHAs and checking cache status..." +echo "Cache version: $CACHE_VERSION" +echo "Force rebuild mode: $FORCE_REBUILD" +if [ -n "$APPS_TO_REBUILD" ]; then + echo "Apps to rebuild: $APPS_TO_REBUILD" +fi +echo "" + +# Setup JFrog CLI if credentials are available +JFROG_AVAILABLE="false" +echo "=== JFrog Setup ===" +echo "JF_URL present: $([ -n "$JF_URL" ] && echo 'YES' || echo 'NO')" +echo "JF_USER present: $([ -n "$JF_USER" ] && echo 'YES' || echo 'NO')" +echo "JF_ACCESS_TOKEN present: $([ -n "$JF_ACCESS_TOKEN" ] && echo 'YES' || echo 'NO')" + +if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "✓ All JFrog credentials available" + if ! command -v jf >/dev/null 2>&1; then + echo "⚠ JFrog CLI (jf) not in PATH; skipping JFrog cache checks" + echo " (jf should be installed by jfrog/setup-jfrog-cli before this script runs)" + else + echo "JFrog CLI version: $(jf --version)" + + # Configure JFrog (skip if already configured — avoids "Server ID already exists" when + # a previous step in the same job has already run jf config add with the same server ID) + if jf config show jfrog-server >/dev/null 2>&1; then + echo "JFrog server 'jfrog-server' already configured, reusing" + else + echo "Configuring JFrog server: $JF_URL" + jf config add jfrog-server --url="$JF_URL" --user="$JF_USER" --access-token="$JF_ACCESS_TOKEN" --interactive=false + fi + + # Test connection with verbose output + echo "Testing JFrog connection..." + if jf rt ping; then + JFROG_AVAILABLE="true" + echo "✓ JFrog connection successful" + echo "Repository: $ARTIFACTORY_REPOSITORY_SNAPSHOT" + else + echo "⚠ JFrog ping failed, will fall back to GitHub cache" + fi + fi +else + echo "⚠ JFrog credentials not available, using GitHub cache only" + [ -z "$JF_URL" ] && echo " - Missing: JF_URL" + [ -z "$JF_USER" ] && echo " - Missing: JF_USER" + [ -z "$JF_ACCESS_TOKEN" ] && echo " - Missing: JF_ACCESS_TOKEN" +fi +echo "JFROG_AVAILABLE=$JFROG_AVAILABLE" +echo "===================" +echo "" + +# Get the matrix from input (passed as argument) +MATRIX="$1" + +# Build JSON arrays for apps that need building/restoring +APPS_TO_BUILD="[]" +APPS_TO_RESTORE="[]" +APPS_CHECKED=0 +APPS_CACHED=0 +APPS_IN_JFROG=0 +APPS_TO_BUILD_COUNT=0 +APPS_TO_RESTORE_COUNT=0 +APPS_SHA_MAP="{}" +echo "" + +echo "### 📦 Cache Status Report for ($GITHUB_REF)" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" +echo "**Cache Version:** \`$CACHE_VERSION\`" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" +if [ "$FORCE_REBUILD" == "true" ]; then + echo "**🔄 FORCE REBUILD MODE ENABLED** - All caches bypassed" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +if [ -n "$APPS_TO_REBUILD" ]; then + echo "**🔨 Specific apps to rebuild:** \`$APPS_TO_REBUILD\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +if [ "$JFROG_AVAILABLE" == "true" ]; then + echo "**🎯 JFrog Artifact Cache**: Enabled for all branches" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi +echo "| App | SHA | Cache Key | Status |" >> "$GITHUB_STEP_SUMMARY" +echo "|-----|-----|-----------|--------|" >> "$GITHUB_STEP_SUMMARY" + +# Convert comma-separated apps list to array for easier checking +if [ -n "$APPS_TO_REBUILD" ]; then + IFS=',' read -ra REBUILD_APPS_ARRAY <<< "$APPS_TO_REBUILD" + # Trim whitespace from each app name using xargs for readability + for i in "${!REBUILD_APPS_ARRAY[@]}"; do + REBUILD_APPS_ARRAY[$i]=$(echo "${REBUILD_APPS_ARRAY[$i]}" | xargs) + done +else + REBUILD_APPS_ARRAY=() +fi + +# Iterate through each app in the matrix +while IFS= read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_PATH=$(echo "$app_json" | jq -r '.path') + + APPS_CHECKED=$((APPS_CHECKED + 1)) + + # Get current submodule SHA + if [ -d "$APP_PATH" ]; then + CURRENT_SHA=$(git -C "$APP_PATH" rev-parse HEAD 2>/dev/null || echo "") + else + echo "⊘ $APP_NAME - directory not found at '$APP_PATH' (submodule not initialised?), will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Directory not found — will build |" >> "$GITHUB_STEP_SUMMARY" + UNKNOWN_SUFFIX="unknown" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$UNKNOWN_SUFFIX" \ + --arg archive_name "${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + if [ -z "$CURRENT_SHA" ]; then + echo "⊘ $APP_NAME - '$APP_PATH' exists but is not a git repo, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Not a git repo — will build |" >> "$GITHUB_STEP_SUMMARY" + UNKNOWN_SUFFIX="unknown" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$UNKNOWN_SUFFIX" \ + --arg archive_name "${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${UNKNOWN_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Add SHA to the map for all apps (regardless of cache status) + APPS_SHA_MAP=$(echo "$APPS_SHA_MAP" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '.[$app] = $sha') + + # Cache key for this app — use 8-char short SHAs throughout. + SHORT_SHA="${CURRENT_SHA:0:8}" + CACHE_KEY="${CACHE_VERSION}-app-build-${APP_NAME}-${SHORT_SHA}" + ARCHIVE_SUFFIX="${SHORT_SHA}" + + echo -n " Checking $APP_NAME (SHA: $SHORT_SHA)... " + + # If force rebuild is enabled, skip cache check and rebuild everything + if [ "$FORCE_REBUILD" == "true" ]; then + echo "🔄 force rebuild" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔄 Force rebuild |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Check if this specific app should be rebuilt (ignore cache) + REBUILD_THIS_APP=false + for rebuild_app in "${REBUILD_APPS_ARRAY[@]}"; do + if [ "$APP_NAME" == "$rebuild_app" ]; then + REBUILD_THIS_APP=true + break + fi + done + + if [ "$REBUILD_THIS_APP" == "true" ]; then + echo "🔨 rebuild requested" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Rebuild requested |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Check JFrog first before GitHub cache (available for all branches) + if [ "$JFROG_AVAILABLE" == "true" ]; then + JFROG_PATH="${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" + FOUND_IN_JFROG="false" + + echo "" + echo " 🔍 Checking JFrog for $APP_NAME..." + echo " Path: $JFROG_PATH" + echo " SHA: $SHORT_SHA" + + # Check if artifact exists in JFrog with verbose output + echo " Running: jf rt s \"$JFROG_PATH\"" + if SEARCH_OUTPUT=$(jf rt s "$JFROG_PATH" 2>&1); then + echo " Search exit code: 0" + echo " Search output:" + echo "$SEARCH_OUTPUT" | sed 's/^/ /' + + if echo "$SEARCH_OUTPUT" | grep -q "$JFROG_PATH"; then + echo " ✓ Artifact found in JFrog!" + ARCHIVE_NAME="${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" + echo "✓ in JFrog" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$JFROG_PATH\` | 📦 In JFrog |" >> "$GITHUB_STEP_SUMMARY" + APPS_IN_JFROG=$((APPS_IN_JFROG + 1)) + APPS_TO_RESTORE_COUNT=$((APPS_TO_RESTORE_COUNT + 1)) + # Add to restore list with JFrog source + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --argjson app "$app_json" --arg sha "$CURRENT_SHA" --arg jfrog_path "$JFROG_PATH" --arg archive_name "$ARCHIVE_NAME" --arg source "jfrog" '. + [($app + {sha: $sha, jfrog_path: $jfrog_path, archive_name: $archive_name, source: $source})]') + FOUND_IN_JFROG="true" + else + echo " ✗ Artifact not found in search results" + fi + else + echo " Search exit code: non-zero" + echo " ✗ Search failed with error:" + echo "$SEARCH_OUTPUT" | sed 's/^/ /' + fi + + if [ "$FOUND_IN_JFROG" == "true" ]; then + continue + fi + + echo "⚡ needs build (JFrog miss)" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$JFROG_PATH\` | 🔨 Needs build (not in JFrog) |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Check if cache exists using GitHub CLI + # Include --ref to access caches from the current ref (branch, PR, etc.) + if ! CACHE_LIST=$(gh cache list --ref "$GITHUB_REF" --key "$CACHE_KEY" --json key --jq ".[].key" 2>&1); then + echo "⚠️ Warning: Failed to query cache for $APP_NAME: $CACHE_LIST" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ⚠️ Cache check failed - will build |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + if echo "$CACHE_LIST" | grep -q "^${CACHE_KEY}$"; then + APPS_CACHED=$((APPS_CACHED + 1)) + APPS_TO_RESTORE_COUNT=$((APPS_TO_RESTORE_COUNT + 1)) + echo "✓ cached" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ✅ Cached |" >> "$GITHUB_STEP_SUMMARY" + # Add to restore list with GitHub cache source + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --argjson app "$app_json" --arg sha "$CURRENT_SHA" --arg cache_key "$CACHE_KEY" --arg source "github-cache" '. + [($app + {sha: $sha, cache_key: $cache_key, source: $source})]') + else + echo "⚡ needs build" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Needs build |" >> "$GITHUB_STEP_SUMMARY" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c \ + --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" \ + --arg archive_name "${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + --arg jfrog_path "${ARTIFACTORY_REPOSITORY_SNAPSHOT}/apps/${CACHE_VERSION}/${APP_NAME}/${APP_NAME}-${ARCHIVE_SUFFIX}.tar.gz" \ + '. + [{name: $app, sha: $sha, archive_name: $archive_name, jfrog_path: $jfrog_path}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + fi + +done < <(echo "$MATRIX" | jq -c '.[]') + +echo "" >> "$GITHUB_STEP_SUMMARY" +echo "**Summary:**" >> "$GITHUB_STEP_SUMMARY" +echo "- Total apps checked: $APPS_CHECKED" >> "$GITHUB_STEP_SUMMARY" +echo "- 📦 Apps in JFrog: $APPS_IN_JFROG" >> "$GITHUB_STEP_SUMMARY" +echo "- ✅ Apps with cached builds: $APPS_CACHED" >> "$GITHUB_STEP_SUMMARY" +echo "- 🔨 Apps needing build: $APPS_TO_BUILD_COUNT" >> "$GITHUB_STEP_SUMMARY" +echo "" >> "$GITHUB_STEP_SUMMARY" + +TOTAL_AVAILABLE=$((APPS_IN_JFROG + APPS_CACHED)) +if [ $TOTAL_AVAILABLE -gt 0 ] && [ $APPS_CHECKED -gt 0 ]; then + CACHE_HIT_PERCENT=$((TOTAL_AVAILABLE * 100 / APPS_CHECKED)) + echo "**Cache hit rate: ${CACHE_HIT_PERCENT}%** 🎯" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" +fi + +echo "" +echo "Summary:" +echo " Total apps: $APPS_CHECKED" +echo " In JFrog: $APPS_IN_JFROG" +echo " Cached: $APPS_CACHED" +echo " To build: $APPS_TO_BUILD_COUNT" + +# Validate no duplicate apps in build and restore lists +BUILD_APPS=$(echo "$APPS_TO_BUILD" | jq -r '.[].name' | sort) +RESTORE_APPS=$(echo "$APPS_TO_RESTORE" | jq -r '.[].name' | sort) +DUPLICATE_APPS=$(comm -12 <(echo "$BUILD_APPS") <(echo "$RESTORE_APPS")) + +if [ -n "$DUPLICATE_APPS" ]; then + echo "ERROR: Apps appear in both build and restore lists:" + echo "$DUPLICATE_APPS" + exit 1 +fi + +# Validate that we built valid JSON +if ! echo "$APPS_TO_BUILD" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_build" + echo "Content: $APPS_TO_BUILD" + exit 1 +fi + +if ! echo "$APPS_TO_RESTORE" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps_to_restore" + echo "Content: $APPS_TO_RESTORE" + exit 1 +fi + +# Output app list with SHAs for the build job to use +# Use proper multiline output format for GitHub Actions +echo "apps_to_build<> "$GITHUB_OUTPUT" +echo "$APPS_TO_BUILD" >> "$GITHUB_OUTPUT" +echo "APPS_TO_BUILD_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output the unified list of apps to restore (from either GitHub cache or JFrog) +echo "apps_to_restore<> "$GITHUB_OUTPUT" +echo "$APPS_TO_RESTORE" >> "$GITHUB_OUTPUT" +echo "APPS_TO_RESTORE_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output the SHA map for all apps +echo "apps_sha_map<> "$GITHUB_OUTPUT" +echo "$APPS_SHA_MAP" >> "$GITHUB_OUTPUT" +echo "APPS_SHA_MAP_JSON_EOF" >> "$GITHUB_OUTPUT" + +# Output flags for conditional job execution +if [ $APPS_TO_BUILD_COUNT -gt 0 ]; then + echo "has_apps_to_build=true" >> "$GITHUB_OUTPUT" +else + echo "has_apps_to_build=false" >> "$GITHUB_OUTPUT" +fi + +if [ $APPS_TO_RESTORE_COUNT -gt 0 ]; then + echo "has_apps_to_restore=true" >> "$GITHUB_OUTPUT" +else + echo "has_apps_to_restore=false" >> "$GITHUB_OUTPUT" +fi + +echo "" +if [ $APPS_TO_BUILD_COUNT -eq 0 ]; then + echo "🎉 All apps are cached! No builds needed." +else + echo "✓ Will build $APPS_TO_BUILD_COUNT app(s)" +fi diff --git a/.github/workflows/hidrive-next-build.yml b/.github/workflows/hidrive-next-build.yml new file mode 100644 index 0000000000000..4a072992352f6 --- /dev/null +++ b/.github/workflows/hidrive-next-build.yml @@ -0,0 +1,1084 @@ +name: HiDrive Next Build + +# SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors +# SPDX-FileCopyrightText: 2024 STRATO AG +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Cached parallel matrix build pipeline: +# Stage 1: prepare-matrix — compute app build matrix from IONOS/Makefile and probe per-app caches +# Stage 2: build-apps — parallel matrix per app, per-app SHA caching in JFrog + GitHub Actions cache +# Final: hidrive-next-build — restore all per-app artifacts and build Nextcloud core + +on: + pull_request: + paths: + - '.github/workflows/**' + - 'src/**' + - 'apps/**' + - 'apps/**/appinfo/info.xml' + - 'apps-custom/**' + - 'apps-external/**' + - 'IONOS' + - 'package.json' + - 'package-lock.json' + - 'themes/**' + - 'lib/**' + - 'tsconfig.json' + - '**.js' + - '**.ts' + - '**.vue' + - '.gitmodules' + push: + branches: + - ionos-dev + - ionos-stable + - 'rc/**' + - '*/dev/*' + workflow_dispatch: + inputs: + force_rebuild: + description: 'Force rebuild all apps and dependencies (bypass ALL caches)' + required: false + type: boolean + default: false + cache_version_suffix: + description: 'Optional cache version suffix (e.g., "test", "debug") - creates separate cache namespace' + required: false + type: string + default: '' + apps_to_rebuild: + description: 'Comma-separated list of specific apps to rebuild (e.g., "simplesettings,viewer")' + required: false + type: string + default: '' + +# Concurrency group is intentionally shared between push and pull_request runs +# of the same source branch, so a `*/dev/*` branch with an open PR does not +# produce two parallel runs. Protected branches (ionos-dev/stable/rc/*) use a +# unique-per-run-id key so consecutive pushes never cancel each other. +concurrency: + group: >- + ${{ github.workflow }}-${{ + ( + contains(fromJson('["refs/heads/ionos-dev","refs/heads/ionos-stable"]'), github.ref) || + startsWith(github.ref, 'refs/heads/rc/') + ) && github.run_id || + (github.head_ref || github.ref_name) + }} + cancel-in-progress: true + +env: + TARGET_PACKAGE_NAME: hidrive-next.zip + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + ARTIFACTORY_REPOSITORY_SNAPSHOT: ionos-productivity-hdnext-snapshot + # Cache version - increment to invalidate all caches when build tooling changes + # Format: v. (e.g., v1.0, v1.1, v2.0) + CACHE_VERSION: v1.2 + +permissions: + contents: read + +jobs: + prepare-matrix: + runs-on: ubuntu-latest + outputs: + apps_to_build: ${{ steps.detect.outputs.apps_to_build }} + apps_to_restore: ${{ steps.detect.outputs.apps_to_restore }} + apps_matrix: ${{ steps.set_matrix.outputs.matrix }} + apps_sha_map: ${{ steps.detect.outputs.apps_sha_map }} + has_apps_to_build: ${{ steps.detect.outputs.has_apps_to_build }} + has_apps_to_restore: ${{ steps.detect.outputs.has_apps_to_restore }} + effective_cache_version: ${{ steps.compute_cache_version.outputs.effective_cache_version }} + + permissions: + contents: read + actions: read + + steps: + - name: Compute effective cache version + id: compute_cache_version + run: | + EFFECTIVE_VERSION="${{ env.CACHE_VERSION }}${{ github.event.inputs.cache_version_suffix && format('-{0}', github.event.inputs.cache_version_suffix) || '' }}" + echo "effective_cache_version=$EFFECTIVE_VERSION" >> "$GITHUB_OUTPUT" + echo "Effective cache version: $EFFECTIVE_VERSION" + + - name: Checkout repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + submodules: true + fetch-depth: 1 + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y make jq + + - name: Check configuration + run: | + echo "### 🔧 Remote Trigger Configuration" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**DISABLE_REMOTE_TRIGGER value:** \`${{ vars.DISABLE_REMOTE_TRIGGER }}\`" >> $GITHUB_STEP_SUMMARY + echo "**ENABLE_REMOTE_TRIGGER_USER_DEV value:** \`${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}\`" >> $GITHUB_STEP_SUMMARY + if [ "${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}" != "true" ]; then + echo " - 💡 To enable the GitLab trigger for \`*/dev/*\` branches, set repository variable \`ENABLE_REMOTE_TRIGGER_USER_DEV\` to \`true\` at [Settings → Variables → Actions](https://github.com/${{ github.repository }}/settings/variables/actions)." >> $GITHUB_STEP_SUMMARY + fi + echo "**Event type:** \`${{ github.event_name }}\`" >> $GITHUB_STEP_SUMMARY + echo "**Branch:** \`${{ github.ref_name }}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "🔧 Remote Trigger Configuration" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "DISABLE_REMOTE_TRIGGER = '${{ vars.DISABLE_REMOTE_TRIGGER }}'" + echo "ENABLE_REMOTE_TRIGGER_USER_DEV = '${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}'" + if [ "${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}" != "true" ]; then + echo " 💡 To enable the GitLab trigger for '*/dev/*' branches," + echo " set repository variable ENABLE_REMOTE_TRIGGER_USER_DEV to 'true' at:" + echo " https://github.com/${{ github.repository }}/settings/variables/actions" + fi + echo "Event type = '${{ github.event_name }}'" + echo "Branch = '${{ github.ref_name }}'" + echo "" + + if [ "${{ vars.DISABLE_REMOTE_TRIGGER }}" == "true" ]; then + echo "⚠️ DISABLE_REMOTE_TRIGGER='true' — remote trigger is force-disabled" + echo " The 'trigger-remote-dev-workflow' job will be SKIPPED" + echo "**Status:** ⚠️ Remote trigger is **force-disabled** via \`DISABLE_REMOTE_TRIGGER='true'\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "The \`trigger-remote-dev-workflow\` job will be skipped." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "To enable, delete the variable or set it to a value other than 'true' at:" >> $GITHUB_STEP_SUMMARY + echo "https://github.com/${{ github.repository }}/settings/variables/actions" >> $GITHUB_STEP_SUMMARY + else + echo "ℹ️ DISABLE_REMOTE_TRIGGER not set — checking trigger conditions..." + echo "**Status:** \`DISABLE_REMOTE_TRIGGER\` not set — actual outcome determined by conditions below." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + WILL_TRIGGER=true + echo "**Trigger Conditions Check:**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "${{ github.event_name }}" != "push" ]; then + echo "- ❌ Event must be 'push' (current: \`${{ github.event_name }}\`)" >> $GITHUB_STEP_SUMMARY + echo " ❌ Event type is '${{ github.event_name }}' (must be 'push')" + WILL_TRIGGER=false + else + echo "- ✅ Event is 'push'" >> $GITHUB_STEP_SUMMARY + echo " ✅ Event type is 'push'" + fi + + VALID_BRANCH_PATTERN='^(ionos-dev|ionos-stable)$|^rc/.*$|^[^/]+/dev/.*$' + USER_DEV_PATTERN='^[^/]+/dev/.*$' + if [[ ! "${{ github.ref_name }}" =~ $VALID_BRANCH_PATTERN ]]; then + echo "- ❌ Branch must be 'ionos-dev', 'ionos-stable', 'rc/*' or '*/dev/*' (current: \`${{ github.ref_name }}\`)" >> $GITHUB_STEP_SUMMARY + echo " ❌ Branch is '${{ github.ref_name }}' (must be 'ionos-dev', 'ionos-stable', 'rc/*' or '*/dev/*')" + WILL_TRIGGER=false + else + echo "- ✅ Branch is '\`${{ github.ref_name }}\`'" >> $GITHUB_STEP_SUMMARY + echo " ✅ Branch is '${{ github.ref_name }}'" + fi + + if [[ "${{ github.ref_name }}" =~ $USER_DEV_PATTERN ]]; then + if [ "${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}" == "true" ]; then + echo "- ✅ User-dev branch opt-in (\`ENABLE_REMOTE_TRIGGER_USER_DEV='true'\`)" >> $GITHUB_STEP_SUMMARY + echo " ✅ ENABLE_REMOTE_TRIGGER_USER_DEV='true' — '*/dev/*' trigger is opted in" + else + echo "- ❌ User-dev branch requires \`ENABLE_REMOTE_TRIGGER_USER_DEV='true'\` (current: \`${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}\`)" >> $GITHUB_STEP_SUMMARY + echo " ❌ '*/dev/*' branch requires ENABLE_REMOTE_TRIGGER_USER_DEV='true' (current: '${{ vars.ENABLE_REMOTE_TRIGGER_USER_DEV }}')" + WILL_TRIGGER=false + fi + fi + + echo "- ℹ️ All dependent jobs must succeed (checked at job runtime)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "$WILL_TRIGGER" = "true" ]; then + echo "**Expected:** The \`trigger-remote-dev-workflow\` job **WILL RUN** (if all dependent jobs succeed)." >> $GITHUB_STEP_SUMMARY + echo "🎯 Expected: trigger-remote-dev-workflow job WILL RUN (if all dependent jobs succeed)" + else + echo "**Expected:** The \`trigger-remote-dev-workflow\` job **WILL BE SKIPPED** due to unmet conditions above." >> $GITHUB_STEP_SUMMARY + echo "⏭️ Expected: trigger-remote-dev-workflow job WILL BE SKIPPED" + fi + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + - name: List caches before restore + run: gh cache list + env: + GH_TOKEN: ${{ github.token }} + + - name: Check JFrog credentials + id: jfrog-available + if: github.event.inputs.force_rebuild != 'true' + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + run: | + if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "available=true" >> $GITHUB_OUTPUT + else + echo "available=false" >> $GITHUB_OUTPUT + echo "⚠ Full JFrog credential set not available — skipping JFrog checks" + fi + + - name: Setup JFrog CLI + if: steps.jfrog-available.outputs.available == 'true' + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Generate apps matrix from Makefile + id: set_matrix + run: | + echo "Generating apps matrix from Makefile..." + matrix_output=$(make -f IONOS/Makefile generate_apps_matrix_json 2>&1) + if echo "$matrix_output" | grep -q '^\[i\]'; then + matrix=$(echo "$matrix_output" | grep -v '^\[i\]') + else + matrix="$matrix_output" + fi + if ! echo "$matrix" | jq empty 2>/dev/null; then + echo "Error: Generated matrix is not valid JSON" + echo "Output: $matrix_output" + exit 1 + fi + echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT + echo "Matrix generated with $(echo "$matrix" | jq 'length') apps" + + - name: Collect apps SHA and check cache status + id: detect + env: + GH_TOKEN: ${{ github.token }} + CACHE_VERSION: ${{ steps.compute_cache_version.outputs.effective_cache_version }} + FORCE_REBUILD: ${{ github.event.inputs.force_rebuild || 'false' }} + APPS_TO_REBUILD: ${{ github.event.inputs.apps_to_rebuild || '' }} + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + ARTIFACTORY_REPOSITORY_SNAPSHOT: ${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }} + GITHUB_REF: ${{ github.ref }} + run: | + bash .github/scripts/detect-app-cache.sh '${{ steps.set_matrix.outputs.matrix }}' + + build-apps: + runs-on: ubuntu-latest + needs: prepare-matrix + if: | + always() && + needs.prepare-matrix.result == 'success' && + needs.prepare-matrix.outputs.has_apps_to_build == 'true' + + permissions: + contents: read + actions: write + + name: build-apps + strategy: + max-parallel: 7 + matrix: + app_info: ${{ fromJson(needs.prepare-matrix.outputs.apps_to_build) }} + + steps: + - name: Get app configuration from full matrix + id: app-config + run: | + FULL_MATRIX='${{ needs.prepare-matrix.outputs.apps_matrix }}' + APP_NAME='${{ matrix.app_info.name }}' + + APP_CONFIG=$(echo "$FULL_MATRIX" | jq -c --arg name "$APP_NAME" '.[] | select(.name == $name)') + if [ -z "$APP_CONFIG" ]; then + echo "ERROR: Could not find configuration for $APP_NAME in matrix" + exit 1 + fi + + echo "path=$(echo "$APP_CONFIG" | jq -r '.path')" >> $GITHUB_OUTPUT + echo "has-npm=$(echo "$APP_CONFIG" | jq -r '.has_npm')" >> $GITHUB_OUTPUT + echo "has-composer=$(echo "$APP_CONFIG" | jq -r '.has_composer')" >> $GITHUB_OUTPUT + echo "npm-lock-path=$(echo "$APP_CONFIG" | jq -r '.npm_lock_path')" >> $GITHUB_OUTPUT + echo "makefile-target=$(echo "$APP_CONFIG" | jq -r '.makefile_target')" >> $GITHUB_OUTPUT + echo "Building $APP_NAME (SHA: ${{ matrix.app_info.sha }})" + + - name: Checkout repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + submodules: true + fetch-depth: 1 + + - name: Check JFrog credentials + id: jfrog-creds + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + run: | + if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "available=true" >> $GITHUB_OUTPUT + else + echo "available=false" >> $GITHUB_OUTPUT + echo "⚠ Full JFrog credential set not available — using GitHub cache path where possible" + fi + + - name: Setup JFrog CLI + if: steps.jfrog-creds.outputs.available == 'true' + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Set up node + if: steps.app-config.outputs.has-npm == 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: "package.json" + cache: 'npm' + cache-dependency-path: ${{ steps.app-config.outputs.npm-lock-path }} + + - name: Setup PHP + if: steps.app-config.outputs.has-composer == 'true' + uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 + with: + php-version: '8.3' + tools: composer:v2 + extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache + env: + runner: ubuntu-latest + + - name: Cache Composer dependencies for ${{ matrix.app_info.name }} + if: steps.app-config.outputs.has-composer == 'true' && github.event.inputs.force_rebuild != 'true' + uses: actions/cache@v4 + with: + path: ${{ steps.app-config.outputs.path }}/vendor + key: ${{ runner.os }}-composer-${{ matrix.app_info.name }}-${{ hashFiles(format('{0}/composer.lock', steps.app-config.outputs.path)) }} + restore-keys: | + ${{ runner.os }}-composer-${{ matrix.app_info.name }}- + + - name: Build ${{ matrix.app_info.name }} + env: + CYPRESS_INSTALL_BINARY: 0 + PUPPETEER_SKIP_DOWNLOAD: true + run: make -f IONOS/Makefile ${{ steps.app-config.outputs.makefile-target }} + + - name: Report build completion + if: success() + run: | + echo "### ✅ Built ${{ matrix.app_info.name }}" >> $GITHUB_STEP_SUMMARY + echo "- **SHA:** \`${{ matrix.app_info.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Path:** ${{ steps.app-config.outputs.path }}" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** Success" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + - name: Compute app cache key + id: app-cache-key + run: | + APP_SHA="${{ matrix.app_info.sha }}" + APP_NAME="${{ matrix.app_info.name }}" + EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}" + SHORT_SHA="${APP_SHA:0:8}" + + echo "cache_key=${EFFECTIVE_VERSION}-app-build-${APP_NAME}-${SHORT_SHA}" >> $GITHUB_OUTPUT + echo "archive_name=${APP_NAME}-${SHORT_SHA}.tar.gz" >> $GITHUB_OUTPUT + + - name: Get Job data + id: get_job_data + continue-on-error: true + uses: ./.github/actions/get-job-data + with: + job-name: 'build-apps (${{ matrix.app_info.name }}, ${{ matrix.app_info.sha }})' + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + + - name: Upload ${{ matrix.app_info.name }} to JFrog + if: steps.jfrog-creds.outputs.available == 'true' + run: | + APP_NAME="${{ matrix.app_info.name }}" + APP_SHA="${{ matrix.app_info.sha }}" + APP_PATH="${{ steps.app-config.outputs.path }}" + EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}" + ARCHIVE_NAME="${{ steps.app-cache-key.outputs.archive_name }}" + JFROG_PATH="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/apps/${EFFECTIVE_VERSION}/${APP_NAME}/${ARCHIVE_NAME}" + + echo "Packaging $APP_NAME..." + tar -czf "$ARCHIVE_NAME" \ + --exclude="node_modules" \ + --exclude=".git" \ + --exclude="*.log" \ + -C "$(dirname "$APP_PATH")" \ + "$(basename "$APP_PATH")" + + echo "Archive size: $(ls -lh "$ARCHIVE_NAME" | awk '{print $5}')" + echo "Uploading to: $JFROG_PATH" + + JFROG_PROPS_LIST=() + JFROG_PROPS_LIST+=("app.name=${APP_NAME}") + JFROG_PROPS_LIST+=("app.sha=${APP_SHA}") + JFROG_PROPS_LIST+=("vcs.branch=${{ github.ref_name }}") + JFROG_PROPS_LIST+=("vcs.revision=${{ github.sha }}") + JOB_URL="${{ steps.get_job_data.outputs.job_html_url }}" + if [ -n "$JOB_URL" ]; then + JFROG_PROPS_LIST+=("job.html_url=${JOB_URL}") + fi + JFROG_PROPS=$(IFS=';'; printf '%s' "${JFROG_PROPS_LIST[*]}") + + if jf rt upload "$ARCHIVE_NAME" "$JFROG_PATH" --target-props "$JFROG_PROPS"; then + echo "✅ Uploaded $APP_NAME to JFrog" + echo "Verifying upload..." + if jf rt s "$JFROG_PATH" 2>/dev/null | grep -q "$JFROG_PATH"; then + echo "✓ Upload verified — artifact is accessible at $JFROG_PATH" + else + echo "⚠ Upload reported success but verification search did not find the artifact" + fi + else + echo "❌ JFrog upload failed" + exit 1 + fi + rm -f "$ARCHIVE_NAME" + + - name: Save build to GitHub Actions cache + uses: actions/cache/save@v4 + with: + path: ${{ steps.app-config.outputs.path }} + key: ${{ steps.app-cache-key.outputs.cache_key }} + + - name: Upload ${{ matrix.app_info.name }} build artifacts + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: app-build-${{ matrix.app_info.name }} + path: | + ${{ steps.app-config.outputs.path }} + !${{ steps.app-config.outputs.path }}/node_modules + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 + + hidrive-next-build: + runs-on: ubuntu-latest + needs: [prepare-matrix, build-apps] + if: | + always() && + needs.prepare-matrix.result == 'success' && + (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') + + permissions: + contents: read + + outputs: + NC_VERSION: ${{ steps.get_nc_version.outputs.NC_VERSION }} + + name: hidrive-next-build + steps: + - name: Checkout server + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + submodules: true + + - name: Check JFrog credentials + id: jfrog-creds + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + run: | + if [ -n "$JF_URL" ] && [ -n "$JF_USER" ] && [ -n "$JF_ACCESS_TOKEN" ]; then + echo "available=true" >> $GITHUB_OUTPUT + else + echo "available=false" >> $GITHUB_OUTPUT + echo "⚠ Full JFrog credential set not available — artifact restore is not possible" + fi + + - name: Assert JFrog credentials are available (required for artifact restore) + if: steps.jfrog-creds.outputs.available != 'true' + run: | + echo "❌ JFrog credentials are required for hidrive-next-build artifact restore." + echo " Set JF_ARTIFACTORY_URL, JF_ARTIFACTORY_USER, and JF_ACCESS_TOKEN secrets." + exit 1 + + - name: Setup JFrog CLI + if: steps.jfrog-creds.outputs.available == 'true' + uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Ping JFrog server + if: steps.jfrog-creds.outputs.available == 'true' + run: jf rt ping + + - name: Restore all apps from JFrog and GitHub cache + if: steps.jfrog-creds.outputs.available == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + EFFECTIVE_VERSION="${{ needs.prepare-matrix.outputs.effective_cache_version }}" + FULL_MATRIX='${{ needs.prepare-matrix.outputs.apps_matrix }}' + + # Restore cached apps (from detect-app-cache.sh output) + if [ "${{ needs.prepare-matrix.outputs.has_apps_to_restore }}" == "true" ]; then + echo "Restoring cached apps..." + APPS_TO_RESTORE='${{ needs.prepare-matrix.outputs.apps_to_restore }}' + while read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_PATH=$(echo "$app_json" | jq -r '.path') + SOURCE=$(echo "$app_json" | jq -r '.source') + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "Restoring: $APP_NAME (source: $SOURCE)" + + if [ "$SOURCE" == "jfrog" ]; then + JFROG_PATH=$(echo "$app_json" | jq -r '.jfrog_path') + ARCHIVE_NAME=$(echo "$app_json" | jq -r '.archive_name // empty') + if [ -z "$ARCHIVE_NAME" ]; then + ARCHIVE_NAME="$(basename "$JFROG_PATH")" + fi + echo "Downloading from JFrog: $JFROG_PATH" + jf rt download "$JFROG_PATH" "$ARCHIVE_NAME" --flat=true + mkdir -p "$(dirname "$APP_PATH")" + tar -xzf "$ARCHIVE_NAME" -C "$(dirname "$APP_PATH")" + rm -f "$ARCHIVE_NAME" + if [ ! -d "$APP_PATH" ]; then + echo "❌ Restore validation failed: $APP_PATH not found" + exit 1 + fi + # appinfo/info.xml lives in standard Nextcloud apps but not in themes/ + case "$APP_PATH" in + apps-*) + if [ ! -f "$APP_PATH/appinfo/info.xml" ]; then + echo "❌ Restore validation failed: $APP_PATH/appinfo/info.xml missing" + exit 1 + fi + ;; + esac + echo "✅ Restored $APP_NAME from JFrog" + + elif [ "$SOURCE" == "github-cache" ]; then + CACHE_KEY=$(echo "$app_json" | jq -r '.cache_key') + echo "❌ Cannot restore $APP_NAME from GitHub cache within a shell step." + echo " Cache key: $CACHE_KEY" + echo " GitHub Actions cache requires 'actions/cache/restore@v4' as a dedicated workflow step." + exit 1 + fi + done < <(echo "$APPS_TO_RESTORE" | jq -c '.[]') + fi + + # Restore newly built apps from JFrog (apps_to_build) + if [ "${{ needs.prepare-matrix.outputs.has_apps_to_build }}" == "true" ]; then + echo "Restoring newly built apps from JFrog..." + APPS_TO_BUILD='${{ needs.prepare-matrix.outputs.apps_to_build }}' + while read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + ARCHIVE_NAME=$(echo "$app_json" | jq -r '.archive_name') + JFROG_PATH=$(echo "$app_json" | jq -r '.jfrog_path') + APP_PATH=$(echo "$FULL_MATRIX" | jq -r --arg name "$APP_NAME" '.[] | select(.name == $name) | .path') + + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "Restoring newly built: $APP_NAME" + echo "Downloading from JFrog: $JFROG_PATH" + + jf rt download "$JFROG_PATH" "$ARCHIVE_NAME" --flat=true + mkdir -p "$(dirname "$APP_PATH")" + tar -xzf "$ARCHIVE_NAME" -C "$(dirname "$APP_PATH")" + rm -f "$ARCHIVE_NAME" + + if [ ! -d "$APP_PATH" ]; then + echo "❌ Restore validation failed: $APP_PATH not found" + exit 1 + fi + # appinfo/info.xml lives in standard Nextcloud apps but not in themes/ + case "$APP_PATH" in + apps-*) + if [ ! -f "$APP_PATH/appinfo/info.xml" ]; then + echo "❌ Restore validation failed: $APP_PATH/appinfo/info.xml missing" + exit 1 + fi + ;; + esac + echo "✅ Restored $APP_NAME from JFrog" + done < <(echo "$APPS_TO_BUILD" | jq -c '.[]') + fi + + echo "✅ All apps restored" + + - name: Set up node with version from package.json's engines + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: "package.json" + cache: ${{ github.event.inputs.force_rebuild != 'true' && 'npm' || '' }} + + - name: Install Dependencies + run: sudo apt-get update && sudo apt-get install -y make zip unzip + + - name: Print dependencies versions + run: make --version && node --version && npm --version + + - name: Setup PHP with PECL extension + uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 + with: + php-version: '8.3' + tools: composer:v2 + extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache + env: + runner: self-hosted + + - name: Print PHP install + run: php -i && php -m + + - name: Build Nextcloud (core only — apps restored from cache) + run: make -f IONOS/Makefile build_nextcloud_only + + - name: Add config partials + run: make -f IONOS/Makefile add_config_partials + + # IONOS Customization: Inject build number for production traceability + # This is specific to IONOS HiDrive Next and not part of upstream Nextcloud + - name: Inject build number + run: | + echo "${{ github.run_number }}" > .buildnumber + echo "✅ Build number injected: ${{ github.run_number }}" + cat .buildnumber + + - name: Zip dependencies + run: make -f IONOS/Makefile zip_dependencies TARGET_PACKAGE_NAME=${{ env.TARGET_PACKAGE_NAME }} + + - name: Get NC version + id: get_nc_version + continue-on-error: false + run: | + NC_VERSION=$(jq -r '.ncVersion' version.json) + echo "NC_VERSION: $NC_VERSION" + + if [ -z "$NC_VERSION" ]; then + echo "NC_VERSION is empty" + exit 1 + fi + + echo "NC_VERSION=$NC_VERSION" >> $GITHUB_OUTPUT + + - name: Upload artifact result for job hidrive-next-build + uses: actions/upload-artifact@v4 + with: + retention-days: 30 + name: hidrive_next_build_artifact + path: ${{ env.TARGET_PACKAGE_NAME }} + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 # make it red to grab attention + + upload-to-artifactory: + runs-on: self-hosted + # Upload the artifact to the Artifactory repository on PR *OR* on + # "ionos-dev|ionos-stable|rc/*|*/dev/*" branch push *OR* on manual workflow_dispatch + if: | + always() && + (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || + github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' || + startsWith(github.ref_name, 'rc/') || contains(github.ref_name, '/dev/')) && + needs.prepare-matrix.result == 'success' && + (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') && + needs.hidrive-next-build.result == 'success' + + name: Push to artifactory + needs: [prepare-matrix, build-apps, hidrive-next-build] + + outputs: + ARTIFACTORY_LAST_BUILD_PATH: ${{ steps.artifactory_upload.outputs.ARTIFACTORY_LAST_BUILD_PATH }} + + env: + BUILD_NAME: "hidrive_next-snapshot" + + steps: + - name: Check prerequisites + run: | + echo "Checking if required secrets are set..." + error_count=0 + + if [ -z "${{ secrets.JF_ARTIFACTORY_URL }}" ]; then + echo "::error::JF_ARTIFACTORY_URL secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ secrets.JF_ARTIFACTORY_USER }}" ]; then + echo "::error::JF_ARTIFACTORY_USER secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ secrets.JF_ACCESS_TOKEN }}" ]; then + echo "::error::JF_ACCESS_TOKEN secret is not set" + error_count=$((error_count + 1)) + fi + + if [ $error_count -ne 0 ]; then + echo "::error::Required secrets are not set. Aborting." + exit 1 + fi + + # Checkout is required to access the local composite action at ./.github/actions/get-job-data + - name: Checkout repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + fetch-depth: 1 + + - name: Download artifact zip + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: hidrive_next_build_artifact + + # This action sets up the JFrog CLI with the Artifactory URL and access token + - uses: jfrog/setup-jfrog-cli@v4 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Ping the JF server + run: | + # Ping the server + jf rt ping + + - name: Get Job data + id: get_job_data + continue-on-error: true + uses: ./.github/actions/get-job-data + with: + job-name: 'Push to artifactory' + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ github.run_id }} + + - name: Upload build to artifactory + id: artifactory_upload + run: | + # Artifactory Build Storage Structure: + # | Branch/Event | Stage Prefix | Artifact Path | + # |------------------|------------------|------------------------------------------------------------------------------| + # | Pull Request | pr | pr/hidrive-next-pr-.zip | + # | ionos-dev | dev | dev/hidrive-next-//hidrive-next-.zip | + # | ionos-stable | stable | stable/hidrive-next-//hidrive-next-.zip | + # | rc/* | | rc//hidrive-next-//hidrive-next-.zip | + # | */dev/* | devs/ | devs//hidrive-next-//hidrive-next-.zip | + + ARTIFACTORY_STAGE_PREFIX="dev" + + if [ -n "${{ github.event.pull_request.number }}" ]; then + ARTIFACTORY_STAGE_PREFIX="pr" + elif [ "${{ github.ref_name }}" == "ionos-stable" ]; then + ARTIFACTORY_STAGE_PREFIX="stable" + elif [[ "${{ github.ref_name }}" =~ ^rc/.*$ ]]; then + ARTIFACTORY_STAGE_PREFIX="${{ github.ref_name }}" + elif [[ "${{ github.ref_name }}" =~ ^.*/dev/.*$ ]]; then + BRANCH_PREFIX=$(echo "${{ github.ref_name }}" | sed 's|/.*||' | sed 's/[^A-Za-z0-9._-]/-/g') + ARTIFACTORY_STAGE_PREFIX="devs/${BRANCH_PREFIX}" + fi + + export PATH_TO_DIRECTORY="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/${ARTIFACTORY_STAGE_PREFIX}" + + if [ -n "${{ github.event.pull_request.number }}" ]; then + # PR uploads keep the flat layout (one slot per PR number, top-level pr/) + PATH_TO_FILE="hidrive-next-pr-${{ github.event.pull_request.number }}.zip" + else + # branch uploads nest under // to preserve every build + SHORT_SHA="${{ github.sha }}" + SHORT_SHA="${SHORT_SHA:0:7}" + NC_VERSION="${{ needs.hidrive-next-build.outputs.NC_VERSION }}" + PATH_TO_FILE="hidrive-next-${NC_VERSION}/${SHORT_SHA}/hidrive-next-${NC_VERSION}.zip" + fi + + export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" + + JFROG_PROPS_LIST=() + JFROG_PROPS_LIST+=("build.nc_version=${{ needs.hidrive-next-build.outputs.NC_VERSION }}") + JFROG_PROPS_LIST+=("vcs.branch=${{ github.ref }}") + JFROG_PROPS_LIST+=("vcs.revision=${{ github.sha }}") + JOB_URL="${{ steps.get_job_data.outputs.job_html_url }}" + if [ -n "$JOB_URL" ]; then + JFROG_PROPS_LIST+=("job.html_url=${JOB_URL}") + fi + JFROG_PROPS=$(IFS=';'; printf '%s' "${JFROG_PROPS_LIST[*]}") + + # Upload with retry logic (3 attempts with exponential backoff: 10s, 20s) + MAX_ATTEMPTS=3 + ATTEMPT=1 + UPLOAD_SUCCESS=false + DELAY_SEC=10 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Upload attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ + --build-name "${{ env.BUILD_NAME }}" \ + --build-number ${{ github.run_number }} \ + --target-props "$JFROG_PROPS" \ + $PATH_TO_LATEST_ARTIFACT; then + UPLOAD_SUCCESS=true + echo "✅ Upload successful on attempt $ATTEMPT" + break + else + echo "⚠️ Upload attempt $ATTEMPT failed" + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting $DELAY_SEC seconds before retry..." + sleep $DELAY_SEC + DELAY_SEC=$((DELAY_SEC * 2)) + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$UPLOAD_SUCCESS" != "true" ]; then + echo "❌ Upload failed after $MAX_ATTEMPTS attempts" + exit 1 + fi + + echo "ARTIFACTORY_LAST_BUILD_PATH=${PATH_TO_LATEST_ARTIFACT}" >> $GITHUB_OUTPUT + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 # make it red to grab attention + + hidirve-next-artifact-to-ghcr_io: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + name: Push artifact to ghcr.io + needs: [prepare-matrix, build-apps, hidrive-next-build] + if: | + always() && + (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || + github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' || + startsWith(github.ref_name, 'rc/') || contains(github.ref_name, '/dev/')) && + needs.prepare-matrix.result == 'success' && + (needs.build-apps.result == 'success' || needs.build-apps.result == 'skipped') && + needs.hidrive-next-build.result == 'success' + + steps: + - name: Download artifact zip + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + with: + name: hidrive_next_build_artifact + + - name: Log in to the Container registry + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 + with: + images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" + + - name: Create Dockerfile + run: | + cat >Dockerfile << EOF + FROM busybox as builder + COPY ./${{ env.TARGET_PACKAGE_NAME }} / + WORKDIR /builder + RUN unzip /${{ env.TARGET_PACKAGE_NAME }} -d /builder + + FROM scratch + WORKDIR /app + VOLUME /app + COPY --from=builder /builder /app + EOF + + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Show changes on failure + if: failure() + run: | + exit 1 # make it red to grab attention + + + trigger-remote-dev-workflow: + runs-on: self-hosted + + name: Trigger remote workflow + needs: [ hidrive-next-build, upload-to-artifactory ] + # Trigger remote build on "ionos-dev|ionos-stable|rc/*|*/dev/*" branch *push* defined in on:push:branches + # Can be disabled entirely via repository variable 'DISABLE_REMOTE_TRIGGER' (set to 'true' to disable) + # The "*/dev/*" branch class is gated by repository variable 'ENABLE_REMOTE_TRIGGER_USER_DEV' + # (default off — set to 'true' to enable once GitLab supports BUILD_TYPE=dev-) + # Configure at: https://github.com/IONOS-Productivity/nc-server/settings/variables/actions + if: | + always() && + github.event_name == 'push' && + (github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' || + startsWith(github.ref_name, 'rc/') || + (contains(github.ref_name, '/dev/') && vars.ENABLE_REMOTE_TRIGGER_USER_DEV == 'true')) && + needs.hidrive-next-build.result == 'success' && + needs.upload-to-artifactory.result == 'success' && + vars.DISABLE_REMOTE_TRIGGER != 'true' + steps: + - name: Check prerequisites + run: | + echo "Checking if all required variables are set..." + error_count=0 + + if [ -z "${{ secrets.GITLAB_TOKEN }}" ]; then + echo "::error::GITLAB_TOKEN secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ secrets.GITLAB_TRIGGER_URL }}" ]; then + echo "::error::GITLAB_TRIGGER_URL secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ needs.hidrive-next-build.outputs.NC_VERSION }}" ]; then + echo "::error::NC_VERSION output from hidrive-next-build job is not set" + error_count=$((error_count + 1)) + else + echo "✓ NC_VERSION: ${{ needs.hidrive-next-build.outputs.NC_VERSION }}" + fi + + if [ -z "${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" ]; then + echo "::error::ARTIFACTORY_LAST_BUILD_PATH output from upload-to-artifactory job is not set" + error_count=$((error_count + 1)) + else + echo "✓ ARTIFACTORY_LAST_BUILD_PATH: ${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" + fi + + if [ -z "${{ github.sha }}" ]; then + echo "::error::github.sha is not set" + error_count=$((error_count + 1)) + else + echo "✓ GITHUB_SHA: ${{ github.sha }}" + fi + + if [ -z "${{ github.run_id }}" ]; then + echo "::error::github.run_id is not set" + error_count=$((error_count + 1)) + else + echo "✓ BUILD_ID: ${{ github.run_id }}" + fi + + if [ $error_count -ne 0 ]; then + echo "::error::Required variables are not set. Aborting." + exit 1 + fi + + echo "✅ All required variables are set" + + - name: Trigger remote workflow + run: | + # Enable command echo for debugging purposes + set -x + + # Branch to GitLab Trigger Mapping (see HDNEXT-1373): + # | ref_name | GITLAB_REF | BUILD_TYPE | + # |--------------|------------|-------------------| + # | ionos-dev | main | dev | + # | ionos-stable | main | stable | + # | rc/* | main | rc | + # | */dev/* | main | dev- | (opt-in via ENABLE_REMOTE_TRIGGER_USER_DEV) + + BUILD_TYPE="dev" + + if [ "${{ github.ref_name }}" == "ionos-stable" ]; then + BUILD_TYPE="stable" + elif [[ "${{ github.ref_name }}" =~ ^rc/ ]]; then + BUILD_TYPE="rc" + elif [[ "${{ github.ref_name }}" =~ ^.*/dev/.*$ ]]; then + BRANCH_PREFIX=$(echo "${{ github.ref_name }}" | sed 's|/.*||' | sed 's/[^A-Za-z0-9._-]/-/g') + BUILD_TYPE="dev-${BRANCH_PREFIX}" + fi + + # Construct source build URL for traceability + SOURCE_BUILD_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + echo "Source Build URL: $SOURCE_BUILD_URL" + + # Trigger GitLab pipeline via webhook with retry logic (3 attempts with exponential backoff) + MAX_ATTEMPTS=3 + ATTEMPT=1 + TRIGGER_SUCCESS=false + DELAY_SEC=5 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Trigger attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if curl \ + --silent \ + --insecure \ + --request POST \ + --fail-with-body \ + -o response.json \ + --form token=${{ secrets.GITLAB_TOKEN }} \ + --form ref="main" \ + --form "variables[GITHUB_SHA]=${{ github.sha }}" \ + --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ + --form "variables[NC_VERSION]=${{ needs.hidrive-next-build.outputs.NC_VERSION }}" \ + --form "variables[BUILD_ID]=${{ github.run_id }}" \ + --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ + --form "variables[SOURCE_BUILD_URL]=${SOURCE_BUILD_URL}" \ + "${{ secrets.GITLAB_TRIGGER_URL }}"; then + TRIGGER_SUCCESS=true + echo "✅ Trigger successful on attempt $ATTEMPT" + break + else + RETCODE="$?" + echo "⚠️ Trigger attempt $ATTEMPT failed with code $RETCODE" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting ${DELAY_SEC} seconds before retry..." + sleep $DELAY_SEC + DELAY_SEC=$((DELAY_SEC * 2)) # Exponential backoff: 5s, 10s, 20s + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$TRIGGER_SUCCESS" != "true" ]; then + echo "❌ Trigger failed after $MAX_ATTEMPTS attempts" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + exit 1 + fi + + # Disable command echo + set +x + + # Print and parse json + echo "json<> $GITHUB_OUTPUT + cat response.json >> $GITHUB_OUTPUT + echo "END" >> $GITHUB_OUTPUT + echo "web_url<> $GITHUB_OUTPUT + cat response.json | jq --raw-output '.web_url' >> $GITHUB_OUTPUT + echo "END" >> $GITHUB_OUTPUT + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 # make it red to grab attention diff --git a/.github/workflows/sbom-matrix.yaml b/.github/workflows/sbom-matrix.yaml new file mode 100644 index 0000000000000..365aedde5f9d7 --- /dev/null +++ b/.github/workflows/sbom-matrix.yaml @@ -0,0 +1,417 @@ +name: SBOM generation (matrix) + +# SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors +# SPDX-FileCopyrightText: 2025 STRATO AG +# SPDX-License-Identifier: AGPL-3.0-or-later + +# ============================================================================ +# SBOM Generation Workflow +# ============================================================================ +# +# This workflow automates the generation and upload of Software Bill of Materials +# (SBOMs) for security and compliance purposes. It processes multiple components +# of the Nextcloud ecosystem in parallel using a matrix strategy. +# +# REQUIREMENTS: +# - Repository secrets: +# * DEPENDENCY_TRACK_API_KEY: API key for Dependency Track instance +# * IONOS_CA: CA certificate for secure communication +# - Repository variables: +# * DEPENDENCY_TRACK_BASE_URL: Base URL of Dependency Track instance +# * DT_OBJECT_*: Project IDs for each component in Dependency Track +# +# TRIGGERS: +# - Push to ionos-stable branch (production) +# +# OUTPUT: +# - CycloneDX SBOM files in JSON format (spec version 1.6) +# - Separate SBOMs for PHP (Composer) and JavaScript (NPM) dependencies +# - Combined SBOMs for components with both dependency types +# - Automatic upload to Dependency Track for vulnerability analysis +# ============================================================================ +# +# This workflow generates Software Bill of Materials (SBOMs) for the Nextcloud server +# and its associated components, including themes and apps. It uses a matrix strategy +# to process multiple components in parallel, generating separate SBOMs for: +# - PHP dependencies (via Composer and CycloneDX) +# - JavaScript dependencies (via NPM and CycloneDX) +# - Combined SBOMs when components have both PHP and NPM dependencies +# +# The generated SBOMs are then uploaded to a Dependency Track instance for +# vulnerability scanning and license compliance monitoring. +# +# Workflow Structure: +# 1. get-version: Extracts the project version from version.php +# 2. setup-matrix: Defines the matrix of components to process +# 3. generate-sbom: Generates SBOMs for each component (runs in parallel) +# 4. upload-sboms: Uploads all generated SBOMs to Dependency Track +# +# Components processed: +# - Main Nextcloud server (root directory) +# - Custom legacy theme (themes/nc-ionos-theme/IONOS) +# - Custom apps (apps-custom/*) +# - External apps (apps-external/*) + +on: + push: + branches: + # Enable once approved + - ionos-stable + +env: + NODE_OPTIONS: "--max-old-space-size=4096" + +jobs: + # Job 1: Extract version information from the repository + # This job retrieves the version.php file and extracts the version string + # to create a consistent project version identifier for all SBOMs + get-version: + runs-on: self-hosted + permissions: + contents: read + name: get-version + + outputs: + project_version: ${{ steps.get-version.outputs.project_version }} + + steps: + - name: Set up PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.1' + + - name: Get version.php + run: curl -o version.php https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}/version.php + + - name: Get version string from version.php + id: get-version + run: | + # Extract version string from PHP file and create project version identifier + VERSION_STRING=$(php -r "include 'version.php'; echo \$OC_VersionString;") + COMMIT_SHA=$(echo "${{ github.sha }}" | cut -c 1-7) + PROJECT_VERSION="v${VERSION_STRING}-${COMMIT_SHA}" + echo "version_string=${VERSION_STRING}" >> $GITHUB_OUTPUT + echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT + echo "project_version=${PROJECT_VERSION}" >> $GITHUB_OUTPUT + echo "Project version: ${PROJECT_VERSION}" + + # Job 2: Define the matrix of components to process + # This job creates a JSON matrix containing all components that need SBOM generation. + # Each component entry includes: + # - name: Component identifier + # - path: Relative path to the component directory + # - has_composer: Whether the component has PHP dependencies + # - has_npm: Whether the component has NPM dependencies + # - composer_output/npm_output: Output filenames for generated SBOMs + # - project_id: Environment variable name for Dependency Track project ID + setup-matrix: + runs-on: self-hosted + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - id: set-matrix + run: | + # Define the matrix of components to process + # Each component entry specifies its dependencies and output configuration + matrix='{ + "component": [ + { + "name": "nextcloud", + "project_name": "HiDrive NEXT Core", + "path": ".", + "composer_output": "bom.nextcloud.composer.json", + "npm_output": "bom.nextcloud.npm.json" + }, + { + "name": "theme-nc-ionos-theme-legacy", + "project_name": "HiDrive NEXT Theme: nc-ionos-theme", + "path": "themes/nc-ionos-theme/IONOS", + "npm_output": "bom.theme-nc-ionos-theme-legacy.json" + }, + { + "name": "app-simplesettings", + "project_name": "HiDrive NEXT App: simplesettings", + "path": "apps-custom/simplesettings", + "composer_output": "bom.app-simplesettings.composer.json", + "npm_output": "bom.app-simplesettings.npm.json" + }, + { + "name": "app-googleanalytics", + "project_name": "HiDrive NEXT App: googleanalytics", + "path": "apps-custom/googleanalytics", + "composer_output": "bom.app-googleanalytics.json" + }, + { + "name": "app-ionos-processes", + "project_name": "HiDrive NEXT App: nc_ionos_processes", + "path": "apps-custom/nc_ionos_processes", + "composer_output": "bom.app-ionos-processes.json" + }, + { + "name": "app-theming", + "project_name": "HiDrive NEXT App: nc_theming", + "path": "apps-custom/nc_theming", + "composer_output": "bom.app-theming.json" + }, + { + "name": "app-viewer", + "project_name": "HiDrive NEXT App: viewer", + "path": "apps-external/viewer", + "composer_output": "bom.app-viewer.composer.json", + "npm_output": "bom.app-viewer.npm.json" + }, + { + "name": "app-user_oidc", + "project_name": "HiDrive NEXT App: user_oidc", + "path": "apps-external/user_oidc", + "composer_output": "bom.app-user_oidc.composer.json", + "npm_output": "bom.app-user_oidc.npm.json" + }, + { + "name": "app-groupquota", + "project_name": "HiDrive NEXT App: groupquota", + "path": "apps-external/groupquota", + "composer_output": "bom.app-groupquota.json" + }, + { + "name": "app-richdocuments", + "project_name": "HiDrive NEXT App: richdocuments", + "path": "apps-external/richdocuments", + "composer_output": "bom.app-richdocuments.composer.json", + "npm_output": "bom.app-richdocuments.npm.json" + }, + { + "name": "app-files_downloadlimit", + "project_name": "HiDrive NEXT App: files_downloadlimit", + "path": "apps-external/files_downloadlimit", + "composer_output": "bom.app-files_downloadlimit.composer.json", + "npm_output": "bom.app-files_downloadlimit.npm.json" + }, + { + "name": "app-serverinfo", + "project_name": "HiDrive NEXT App: serverinfo", + "path": "apps-external/serverinfo", + "composer_output": "bom.app-serverinfo.json" + } + ] + }' + echo "matrix=$(echo $matrix | jq -c .)" >> $GITHUB_OUTPUT + + # Job 3: Generate SBOMs for each component (runs in parallel) + # This job uses the matrix strategy to process each component defined in setup-matrix. + # For each component, it: + # 1. Sets up the appropriate runtime environment (PHP and/or Node.js) + # 2. Installs SBOM generation tools (CycloneDX for Composer and NPM) + # 3. Generates SBOMs in CycloneDX format (JSON, spec version 1.6) + # 4. For components with both PHP and NPM dependencies, creates a combined SBOM + # 5. Uploads the generated SBOMs as workflow artifacts + generate-sbom: + runs-on: self-hosted + permissions: + contents: read + name: generate-sbom + needs: [get-version, setup-matrix] + + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup-matrix.outputs.matrix) }} + + steps: + - name: Checkout server + uses: actions/checkout@v5 + with: + submodules: true + + - name: Setup PHP with PECL extension + if: matrix.component.composer_output + uses: shivammathur/setup-php@v2 + with: + tools: composer:v2 + extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache + env: + runner: self-hosted + + - name: Install CycloneDX (Composer) + if: matrix.component.composer_output + run: | + composer global config --no-plugins allow-plugins.cyclonedx/cyclonedx-php-composer true + composer global require cyclonedx/cyclonedx-php-composer:^v5.2.3 + + - name: Generate SBOM (Composer) + if: matrix.component.composer_output + run: | + parent_dir=$(pwd) + echo "parent_dir=${parent_dir}" + cd "${{ matrix.component.path }}" + echo "PWD: $(pwd)" + composer CycloneDX:make-sbom --output-file="${parent_dir}/${{ matrix.component.composer_output }}" --output-format=JSON --spec-version=1.6 + ls -la "${parent_dir}/${{ matrix.component.composer_output }}" + + - name: Set up Node.js + if: matrix.component.npm_output + uses: actions/setup-node@v4 + with: + node-version-file: "${{ matrix.component.path }}/package.json" + cache: 'npm' + cache-dependency-path: "${{ matrix.component.path }}/package-lock.json" + + - name: Install NPM dependencies + if: matrix.component.npm_output + working-directory: ${{ matrix.component.path }} + run: | + npm ci + + - name: Generate SBOM (NPM) + if: matrix.component.npm_output + run: | + parent_dir=$(pwd) + cd "${{ matrix.component.path }}" + npx @cyclonedx/cyclonedx-npm --ignore-npm-errors --package-lock-only --output-format JSON --spec-version 1.6 --output-file "${parent_dir}/${{ matrix.component.npm_output }}" + ls -la "${parent_dir}/${{ matrix.component.npm_output }}" + + - name: Upload component SBOM artifacts + uses: actions/upload-artifact@v4 + with: + name: sbom-${{ matrix.component.name }} + path: | + ${{ matrix.component.composer_output }} + ${{ matrix.component.npm_output }} + if-no-files-found: ignore + + # Job 4: Upload SBOMs to Dependency Track + # This job downloads all SBOM artifacts generated by the previous job and uploads + # them to a Dependency Track instance for vulnerability scanning and analysis. + # The upload only happens for specific branches (ionos-stable and the feature branch). + # + # The job: + # 1. Downloads all SBOM artifacts from the generate-sbom job + # 2. Iterates through each component in the matrix + # 3. Determines which SBOM file to upload (combined, composer-only, or npm-only) + # 4. Uploads each SBOM to the corresponding Dependency Track project + # 5. Uses custom CA certificate for secure communication with Dependency Track + upload-sboms: + needs: [ get-version, setup-matrix, generate-sbom ] + runs-on: self-hosted + + steps: + - name: Download all SBOM artifacts + uses: actions/download-artifact@v5 + with: + pattern: sbom-* + merge-multiple: true + + - name: List downloaded files + run: | + ls -la *.json || echo "No BOM JSON files found" + + - name: Upload SBOMs to Dependency Track + if: github.ref == 'refs/heads/ionos-stable' + env: + DT_BASE_URL: ${{ vars.DEPENDENCY_TRACK_BASE_URL }} + DT_API_KEY: ${{ secrets.DEPENDENCY_TRACK_API_KEY }} + DT_ROOT_OBJECT_ID: ${{ vars.DT_ROOT_OBJECT_ID }} + IONOS_CA_CERT: ${{ secrets.IONOS_CA }} + MATRIX_CONTEXT: ${{ needs.setup-matrix.outputs.matrix }} + PROJECT_VERSION: ${{ needs.get-version.outputs.project_version }} + run: | + # Create temporary CA cert file + cert_file=$(mktemp) + echo "$IONOS_CA_CERT" > "${cert_file}" + + echo "Beginning SBOM upload process..." + echo "PROJECT_VERSION: $PROJECT_VERSION" + + # Function to upload SBOM to Dependency Track via REST API + upload_bom() { + local bom_file="$1" + local project_name="$2" + local artifact_type="$3" + local qualified_artifact_name="$2 $3" + + if [[ ! -f "$bom_file" ]]; then + echo "Warning: $bom_file not found, skipping..." + return 0 + fi + + echo "Uploading SBOM ($bom_file) to parent project ${DT_ROOT_OBJECT_ID} ($project_name and artifact type $artifact_type)..." + + response=$(curl \ + --cacert "${cert_file}" \ + --fail-with-body \ + --silent \ + --show-error \ + --write-out "HTTPSTATUS:%{http_code}" \ + -X POST "${DT_BASE_URL}/api/v1/bom" \ + -H "Content-Type: multipart/form-data" \ + -H "X-API-Key: ${DT_API_KEY}" \ + -F "isLatest=true" \ + -F "autoCreate=true" \ + -F "parentUUID=${DT_ROOT_OBJECT_ID}" \ + -F "projectName=${qualified_artifact_name}" \ + -F "projectTags=hidrive_next,nextcloud" \ + -F "projectVersion=${PROJECT_VERSION}" \ + -F "bom=@${bom_file}") + + http_code=$(echo "$response" | grep -o "HTTPSTATUS:[0-9]*" | cut -d: -f2) + body=$(echo "$response" | sed -E 's/HTTPSTATUS:[0-9]*$//') + + if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then + echo "✓ Successfully uploaded $bom_file" + else + echo "✗ Failed to upload $bom_file (HTTP $http_code)" + echo "Response: $body" + return 1 + fi + } + + # Upload all BOMs by iterating through the matrix components + # Each component is processed to determine which SBOM file to upload + upload_failed=false + + # Use a temporary file to track failures across subshell boundary + failure_file=$(mktemp) + + echo "${MATRIX_CONTEXT}" | jq -r '.component[] | @base64' | while read -r i; do + echo "Processing component from matrix context..." + component_json=$(echo "$i" | base64 --decode) + + composer_output=$(echo "$component_json" | jq -r '.composer_output') + npm_output=$(echo "$component_json" | jq -r '.npm_output') + component_name=$(echo "$component_json" | jq -r '.name') + project_name=$(echo "$component_json" | jq -r '.project_name') + + echo "Processing component: ${component_name}" + + if [[ "$composer_output" == "null" ]] && [[ "$npm_output" == "null" ]]; then + echo "Skipping component with neither composer nor npm." + continue + fi + + if [[ "$composer_output" != "null" ]]; then + bom_file=$(echo "$component_json" | jq -r '.composer_output') + if ! upload_bom "$bom_file" "$project_name" "composer"; then + echo "Failed to upload composer SBOM for $component_name" + echo "UPLOAD_FAILED" > "$failure_file" + fi + fi + + if [[ "$npm_output" != "null" ]]; then + bom_file=$(echo "$component_json" | jq -r '.npm_output') + if ! upload_bom "$bom_file" "$project_name" "npm"; then + echo "Failed to upload npm SBOM for $component_name" + echo "UPLOAD_FAILED" > "$failure_file" + fi + fi + done + + # Check if any uploads failed by reading the temporary file + if [[ -f "$failure_file" && -s "$failure_file" ]]; then + echo "One or more SBOM uploads failed" + rm -f "$failure_file" "${cert_file}" + exit 1 + fi + + # Cleanup + rm -f "$failure_file" "${cert_file}" + + echo "All SBOMs uploaded successfully!" diff --git a/.gitignore b/.gitignore index d2958f124e0ec..9be23f65e8734 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ node_modules/ /translationfiles /translationtool.phar +# CI/CD generated files +/.buildnumber + # ignore all apps except core ones /apps*/* !/apps/cloud_federation_api @@ -54,6 +57,13 @@ node_modules/ /apps/files_external/3rdparty/irodsphp/prods/tutorials /apps/files_external/3rdparty/irodsphp/prods/test* /apps/files_external/tests/config.*.php +# IONOS: this should not be upstreamed! +!/apps-external/viewer +!/apps-external/user_oidc +!/apps-external/serverinfo +!/apps-external/groupquota +!/apps-external/files_downloadlimit +!/apps-custom/* # ignore themes except the example and the README /themes/* @@ -130,6 +140,7 @@ nbproject # nodejs /build/bin /build/lib/ +/build/jsdocs/ /build/integration/output/ /build/integration/phpserver.log /npm-debug.log diff --git a/.gitmodules b/.gitmodules index 61195b5d6fd81..762ee80885526 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,39 @@ [submodule "3rdparty"] path = 3rdparty url = https://github.com/nextcloud/3rdparty.git +[submodule "apps-custom/simplesettings"] + path = apps-custom/simplesettings + url = git@github.com:IONOS-Productivity/nc-simplesettings.git +[submodule "apps-custom/googleanalytics"] + path = apps-custom/googleanalytics + url = git@github.com:IONOS-Productivity/nc-googleanalytics.git +[submodule "apps-custom/nc_theming"] + path = apps-custom/nc_theming + url = git@github.com:IONOS-Productivity/nc-theming.git +[submodule "apps-custom/nc_ionos_processes"] + path = apps-custom/nc_ionos_processes + url = git@github.com:IONOS-Productivity/nc-ionos-processes.git +[submodule "themes/nc-ionos-theme"] + path = themes/nc-ionos-theme + url = git@github.com:IONOS-Productivity/nc-ionos-theme.git +[submodule "apps-external/viewer"] + path = apps-external/viewer + url = git@github.com:IONOS-Productivity/nc-viewer.git +[submodule "apps-external/richdocuments"] + path = apps-external/richdocuments + url = git@github.com:IONOS-Productivity/nc-richdocuments.git +[submodule "apps-external/user_oidc"] + path = apps-external/user_oidc + url = git@github.com:nextcloud/user_oidc.git +[submodule "apps-external/serverinfo"] + path = apps-external/serverinfo + url = git@github.com:nextcloud/serverinfo.git +[submodule "apps-external/groupquota"] + path = apps-external/groupquota + url = git@github.com:nextcloud/groupquota.git +[submodule "apps-external/files_downloadlimit"] + path = apps-external/files_downloadlimit + url = git@github.com:nextcloud/files_downloadlimit.git +[submodule "IONOS"] + path = IONOS + url = git@github.com:IONOS-Productivity/nc-config.git diff --git a/IONOS b/IONOS new file mode 160000 index 0000000000000..ecde9e284b724 --- /dev/null +++ b/IONOS @@ -0,0 +1 @@ +Subproject commit ecde9e284b724e290846f30e7d0dd1b1a0495648 diff --git a/apps-custom/.gitkeep b/apps-custom/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/apps-custom/googleanalytics b/apps-custom/googleanalytics new file mode 160000 index 0000000000000..8766e392f60cb --- /dev/null +++ b/apps-custom/googleanalytics @@ -0,0 +1 @@ +Subproject commit 8766e392f60cb07f0442e4564e14a3463f6dceca diff --git a/apps-custom/nc_ionos_processes b/apps-custom/nc_ionos_processes new file mode 160000 index 0000000000000..170d1705a6c3b --- /dev/null +++ b/apps-custom/nc_ionos_processes @@ -0,0 +1 @@ +Subproject commit 170d1705a6c3b2f09d5270682aca6864497d1afc diff --git a/apps-custom/nc_theming b/apps-custom/nc_theming new file mode 160000 index 0000000000000..a06a89fb5281b --- /dev/null +++ b/apps-custom/nc_theming @@ -0,0 +1 @@ +Subproject commit a06a89fb5281b2fe4b0a1bc23d7ec5ddbaa2cdf6 diff --git a/apps-custom/simplesettings b/apps-custom/simplesettings new file mode 160000 index 0000000000000..373db5b384e72 --- /dev/null +++ b/apps-custom/simplesettings @@ -0,0 +1 @@ +Subproject commit 373db5b384e7200ff87867ffd254be9c2b064862 diff --git a/apps-external/.gitkeep b/apps-external/.gitkeep new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/apps-external/files_downloadlimit b/apps-external/files_downloadlimit new file mode 160000 index 0000000000000..cd703cb1d1845 --- /dev/null +++ b/apps-external/files_downloadlimit @@ -0,0 +1 @@ +Subproject commit cd703cb1d18456292fb1ae60d44107158a387a73 diff --git a/apps-external/groupquota b/apps-external/groupquota new file mode 160000 index 0000000000000..5b2d66eb05046 --- /dev/null +++ b/apps-external/groupquota @@ -0,0 +1 @@ +Subproject commit 5b2d66eb0504671feee3fd8ebdd8ca16504725cf diff --git a/apps-external/richdocuments b/apps-external/richdocuments new file mode 160000 index 0000000000000..69199dc9fc07c --- /dev/null +++ b/apps-external/richdocuments @@ -0,0 +1 @@ +Subproject commit 69199dc9fc07ceb38ecfe6f01c67a9d996504981 diff --git a/apps-external/serverinfo b/apps-external/serverinfo new file mode 160000 index 0000000000000..06f355d7e6d96 --- /dev/null +++ b/apps-external/serverinfo @@ -0,0 +1 @@ +Subproject commit 06f355d7e6d966ee97b9269f130980b6aabcdaa3 diff --git a/apps-external/user_oidc b/apps-external/user_oidc new file mode 160000 index 0000000000000..b5a67c7c6454e --- /dev/null +++ b/apps-external/user_oidc @@ -0,0 +1 @@ +Subproject commit b5a67c7c6454ec6f3872ed424283b788cbbae86b diff --git a/apps-external/viewer b/apps-external/viewer new file mode 160000 index 0000000000000..9e3b79a31742b --- /dev/null +++ b/apps-external/viewer @@ -0,0 +1 @@ +Subproject commit 9e3b79a31742baade17f73ffae00a4f934a74346 diff --git a/apps/files/src/components/SidebarQuota.vue b/apps/files/src/components/SidebarQuota.vue new file mode 100644 index 0000000000000..195ed8c8cb472 --- /dev/null +++ b/apps/files/src/components/SidebarQuota.vue @@ -0,0 +1,161 @@ + + + + + + + diff --git a/apps/files/src/views/FilesNavigation.vue b/apps/files/src/views/FilesNavigation.vue index 293e4176bb019..6df96cb501736 100644 --- a/apps/files/src/views/FilesNavigation.vue +++ b/apps/files/src/views/FilesNavigation.vue @@ -24,7 +24,7 @@