Tutorial Tests Automation #328
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tutorial Tests Automation | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| on: | |
| # Trigger when translation PRs are merged | |
| pull_request: | |
| types: [ closed ] | |
| branches: [ main ] | |
| # Allow manual triggering | |
| workflow_dispatch: | |
| inputs: | |
| tutorial_filter: | |
| description: 'Tutorial ID to test (leave empty for all)' | |
| required: false | |
| type: string | |
| languages: | |
| description: 'Languages to test (space-separated)' | |
| default: 'pt-BR es-419' | |
| required: false | |
| type: string | |
| use_github_tutorialmaker: | |
| description: 'Use TutorialMaker from GitHub instead of Extension Manager' | |
| required: false | |
| type: boolean | |
| default: false | |
| env: | |
| SLICER_VERSION: '5.10.0' | |
| DEFAULT_LANGUAGES: 'pt-BR es-419' | |
| SCREEN_RESOLUTION: '1920x1080' | |
| PIP_NO_CACHE_DIR: '1' | |
| PYTHONUNBUFFERED: '1' | |
| jobs: | |
| detect-tutorials: | |
| name: Detect Tutorials | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tutorials: ${{ steps.scan.outputs.tutorials }} | |
| tutorial-count: ${{ steps.scan.outputs.tutorial-count }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Scan for tutorials with IDs | |
| id: scan | |
| run: | | |
| echo "Scanning for tutorials with valid IDs..." | |
| tutorials_json="[" | |
| tutorial_count=0 | |
| for tutorial_dir in Tutorials/*/; do | |
| if [[ -d "$tutorial_dir" ]]; then | |
| tutorial_name=$(basename "$tutorial_dir") | |
| echo "Checking tutorial: $tutorial_name" | |
| # Extract ID from tutorial name (everything before first underscore) | |
| if [[ "$tutorial_name" =~ ^([^_]+)_ ]]; then | |
| tutorial_id="${BASH_REMATCH[1]}" | |
| # Check if this tutorial should be filtered | |
| if [[ -n "${{ github.event.inputs.tutorial_filter }}" ]]; then | |
| if [[ "$tutorial_id" != "${{ github.event.inputs.tutorial_filter }}" ]]; then | |
| echo " Skipping $tutorial_id (filtered out)" | |
| continue | |
| fi | |
| fi | |
| echo " Found valid tutorial: $tutorial_id -> $tutorial_name" | |
| # Detect available languages for this tutorial | |
| translations_dir="${tutorial_dir}Translations" | |
| available_languages="" | |
| if [[ -d "$translations_dir" ]]; then | |
| echo " Checking translations in: $translations_dir" | |
| # First, check if default translation file exists | |
| default_file="${translations_dir}/text_dict_default.json" | |
| if [[ ! -f "$default_file" ]]; then | |
| echo " ⚠️ No default translation file found, skipping translation check" | |
| else | |
| # Count keys in default file with non-empty values (baseline for 100%) | |
| # Use Python JSON parsing to avoid grep-based false positives | |
| default_keys_count=$(python3 -c "import json,sys; data=json.load(open(sys.argv[1], encoding='utf-8')); norm=lambda v: str(v).strip() if v is not None else ''; print(sum(1 for v in data.values() if norm(v)))" "$default_file") | |
| echo " Default translation has $default_keys_count keys (with non-empty values)" | |
| # Calculate minimum required keys (80% threshold, rounded up) | |
| min_required_keys=$(( (default_keys_count * 80 + 99) / 100 )) | |
| echo " Minimum required keys for 80%: $min_required_keys" | |
| fi | |
| # Find all text_dict_*.json files except default | |
| for dict_file in "$translations_dir"/text_dict_*.json; do | |
| if [[ -f "$dict_file" ]]; then | |
| filename=$(basename "$dict_file") | |
| # Extract language code (between text_dict_ and .json) | |
| if [[ "$filename" =~ ^text_dict_(.+)\.json$ ]]; then | |
| lang_code="${BASH_REMATCH[1]}" | |
| # Skip default | |
| if [[ "$lang_code" != "default" ]]; then | |
| # Always include en-US (English) | |
| if [[ "$lang_code" == "en-US" ]]; then | |
| echo " ✅ en-US is always included (English)" | |
| if [[ -z "$available_languages" ]]; then | |
| available_languages="$lang_code" | |
| else | |
| if [[ ! " $available_languages " =~ " $lang_code " ]]; then | |
| available_languages="$available_languages $lang_code" | |
| fi | |
| fi | |
| # Check if translation is at least 80% complete and not equal to English default values | |
| elif [[ -f "$default_file" ]]; then | |
| # Read translation metrics from Python: | |
| # default_keys_count lang_non_empty_count lang_different_count same_as_default_count | |
| metrics=$(python3 -c "import json,sys; default=json.load(open(sys.argv[1], encoding='utf-8')); lang=json.load(open(sys.argv[2], encoding='utf-8')); norm=lambda v: str(v).strip() if v is not None else ''; keys=[k for k,v in default.items() if norm(v)]; vals=[norm(lang.get(k,'')) for k in keys]; defs=[norm(default.get(k,'')) for k in keys]; non_empty=sum(1 for v in vals if v); same=sum(1 for v,d in zip(vals,defs) if v and v==d); diff=sum(1 for v,d in zip(vals,defs) if v and v!=d); print(len(keys), non_empty, diff, same)" "$default_file" "$dict_file") | |
| read -r default_keys_count lang_non_empty_count lang_different_count same_as_default_count <<< "$metrics" | |
| # Calculate completion percentage based on keys translated to values different from English | |
| if [[ $default_keys_count -gt 0 ]]; then | |
| completion_pct=$(python3 -c "import sys; numerator=int(sys.argv[1]); denominator=int(sys.argv[2]); print(f'{(numerator * 100 / denominator):.1f}')" "$lang_different_count" "$default_keys_count") | |
| echo " $lang_code: $lang_different_count/$default_keys_count keys translated and different from English ($completion_pct%)" | |
| echo " Non-empty values: $lang_non_empty_count | Equal to English: $same_as_default_count" | |
| # Include only if >= 80% translated and no value remains identical to English | |
| if [[ $lang_different_count -ge $min_required_keys && $same_as_default_count -eq 0 ]]; then | |
| echo " ✅ $lang_code is sufficiently translated (>= 80% and all translated values differ from English)" | |
| if [[ -z "$available_languages" ]]; then | |
| available_languages="$lang_code" | |
| else | |
| # Check if not already added (avoid duplicates) | |
| if [[ ! " $available_languages " =~ " $lang_code " ]]; then | |
| available_languages="$available_languages $lang_code" | |
| fi | |
| fi | |
| else | |
| echo " ⚠️ $lang_code skipped (needs >= 80% translated-different values and 0 values equal to English)" | |
| fi | |
| fi | |
| else | |
| # No default file, include all languages (fallback behavior) | |
| if [[ -z "$available_languages" ]]; then | |
| available_languages="$lang_code" | |
| else | |
| if [[ ! " $available_languages " =~ " $lang_code " ]]; then | |
| available_languages="$available_languages $lang_code" | |
| fi | |
| fi | |
| fi | |
| fi | |
| fi | |
| fi | |
| done | |
| fi | |
| # Display available languages (note: en-US is always included separately) | |
| if [[ -z "$available_languages" ]]; then | |
| echo " No additional translations found (en-US will be included automatically)" | |
| else | |
| echo " Additional languages: $available_languages" | |
| fi | |
| if [[ $tutorial_count -gt 0 ]]; then | |
| tutorials_json="$tutorials_json," | |
| fi | |
| tutorials_json="$tutorials_json{\"id\":\"$tutorial_id\",\"name\":\"$tutorial_name\",\"path\":\"$tutorial_dir\",\"languages\":\"$available_languages\"}" | |
| tutorial_count=$((tutorial_count + 1)) | |
| else | |
| echo " Skipping $tutorial_name (no valid ID pattern)" | |
| fi | |
| fi | |
| done | |
| tutorials_json="$tutorials_json]" | |
| echo "Found $tutorial_count tutorials with valid IDs" | |
| echo "tutorials=$tutorials_json" >> $GITHUB_OUTPUT | |
| echo "tutorial-count=$tutorial_count" >> $GITHUB_OUTPUT | |
| echo "Tutorials JSON: $tutorials_json" | |
| test-tutorials: | |
| name: Test Tutorial | |
| runs-on: ubuntu-latest | |
| needs: detect-tutorials | |
| if: | | |
| needs.detect-tutorials.outputs.tutorial-count > 0 && | |
| ( | |
| github.event_name == 'workflow_dispatch' || | |
| github.event_name == 'schedule' || | |
| (github.event_name == 'pull_request' && | |
| github.event.pull_request.merged == true && | |
| contains(github.event.pull_request.labels.*.name, 'translations')) | |
| ) | |
| container: | |
| image: ubuntu:22.04 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| tutorial: ${{ fromJson(needs.detect-tutorials.outputs.tutorials) }} | |
| steps: | |
| - name: Install git for checkout | |
| run: | | |
| apt-get update -qq | |
| apt-get install -y git | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install system dependencies | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| TZ: America/New_York | |
| run: | | |
| # Configure timezone non-interactively | |
| ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone | |
| apt-get update -qq | |
| # Install Slicer prerequisites | |
| apt-get install -y \ | |
| libglu1-mesa \ | |
| libpulse-mainloop-glib0 \ | |
| libnss3 \ | |
| libasound2 \ | |
| qt5dxcb-plugin \ | |
| libsm6 \ | |
| libxcb-icccm4 \ | |
| libxcb-image0 \ | |
| libxcb-keysyms1 \ | |
| libxcb-render-util0 | |
| # Install additional dependencies | |
| apt-get install -y \ | |
| wget curl unzip \ | |
| xvfb x11vnc x11-xserver-utils \ | |
| python3 python3-pip \ | |
| libgl1-mesa-glx \ | |
| libxrender1 libxext6 libice6 \ | |
| libfontconfig1 libxss1 \ | |
| libglib2.0-0 libxrandr2 libxcomposite1 \ | |
| libxdamage1 libxcursor1 libxi6 libxtst6 \ | |
| ca-certificates \ | |
| git bc | |
| - name: Setup Python | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install requests | |
| # Pre-download common packages to avoid issues during test | |
| python3 -m pip install --no-cache-dir numpy || echo "numpy install optional" | |
| - name: Check available resources | |
| run: | | |
| echo "=== System Resources ===" | |
| echo "Memory:" | |
| free -h | |
| echo "" | |
| echo "Disk space:" | |
| df -h | |
| echo "" | |
| echo "CPU info:" | |
| nproc | |
| lscpu | grep "Model name" || echo "CPU info not available" | |
| - name: Cache Slicer installation | |
| id: cache-slicer | |
| uses: actions/cache@v4 | |
| with: | |
| path: /opt/slicer | |
| key: slicer-${{ env.SLICER_VERSION }}-${{ runner.os }} | |
| - name: Download and install Slicer | |
| if: steps.cache-slicer.outputs.cache-hit != 'true' | |
| run: | | |
| echo "Downloading Slicer ${{ env.SLICER_VERSION }}..." | |
| mkdir -p /opt/slicer | |
| cd /opt/slicer | |
| # Download Slicer 5.10.0 | |
| wget -O slicer.tar.gz "https://download.slicer.org/bitstream/6911b598ac7b1c95e7934427" | |
| # Extract | |
| tar -xzf slicer.tar.gz --strip-components=1 | |
| rm slicer.tar.gz | |
| # Make executable | |
| chmod +x ./Slicer | |
| # Verify installation | |
| ls -la /opt/slicer/ | |
| echo "Slicer installation completed" | |
| - name: Verify cached Slicer | |
| if: steps.cache-slicer.outputs.cache-hit == 'true' | |
| run: | | |
| echo "✅ Using cached Slicer installation" | |
| ls -la /opt/slicer/ | |
| chmod +x /opt/slicer/Slicer | |
| - name: Clean cached extensions (force fresh install) | |
| run: | | |
| echo "Removing cached Slicer extensions to ensure fresh installation..." | |
| # Remove extensions directories | |
| if [[ -d "/opt/slicer/slicer.org" ]]; then | |
| echo "Found extensions directory: /opt/slicer/slicer.org" | |
| rm -rf /opt/slicer/slicer.org | |
| echo "✅ Removed cached extensions" | |
| else | |
| echo "No cached extensions found (this is fine)" | |
| fi | |
| # List remaining directories | |
| echo "Slicer directory after cleanup:" | |
| ls -la /opt/slicer/ | head -20 | |
| - name: Setup virtual display | |
| run: | | |
| echo "Setting up virtual display ${{ env.SCREEN_RESOLUTION }}..." | |
| export DISPLAY=:99 | |
| # Start Xvfb with specified resolution | |
| Xvfb :99 -screen 0 ${{ env.SCREEN_RESOLUTION }}x24 > /dev/null 2>&1 & | |
| sleep 3 | |
| # Verify display | |
| echo "DISPLAY=$DISPLAY" >> $GITHUB_ENV | |
| - name: Install required Slicer extensions | |
| run: | | |
| echo "Installing required Slicer extensions..." | |
| export DISPLAY=:99 | |
| # Run the extension installation script | |
| /opt/slicer/Slicer --no-splash --python-script Scripts/install-slicer-extensions.py | |
| echo "Slicer extensions installation completed" | |
| - name: Override TutorialMaker with GitHub version | |
| if: github.event.inputs.use_github_tutorialmaker == 'true' | |
| run: | | |
| echo "🔄 Replacing TutorialMaker extension with GitHub version..." | |
| # Find TutorialMaker qt-scripted-modules directory (where the actual module code is) | |
| QT_MODULES_DIR=$(find /opt/slicer/slicer.org/Extensions-*/TutorialMaker/lib/Slicer-*/qt-scripted-modules -type d | head -n 1) | |
| if [ -z "$QT_MODULES_DIR" ]; then | |
| echo "❌ TutorialMaker qt-scripted-modules directory not found" | |
| exit 1 | |
| fi | |
| echo "Found TutorialMaker qt-scripted-modules at: $QT_MODULES_DIR" | |
| # Clone GitHub repository | |
| echo "Cloning TutorialMaker from GitHub..." | |
| git clone https://github.com/SoniaPujolLab/SlicerTutorialMaker.git /tmp/SlicerTutorialMaker | |
| # Copy TutorialMaker module files to qt-scripted-modules (overwrite) | |
| echo "Copying TutorialMaker module to qt-scripted-modules..." | |
| cp -rf /tmp/SlicerTutorialMaker/TutorialMaker/* "$QT_MODULES_DIR/" | |
| echo "✅ TutorialMaker replaced with GitHub version" | |
| echo "Contents of qt-scripted-modules:" | |
| ls -la "$QT_MODULES_DIR" | |
| - name: Prepare tutorial test environment | |
| shell: bash | |
| run: | | |
| echo "Preparing test environment for tutorial: ${{ matrix.tutorial.name }}" | |
| # Clear pip cache to free up space | |
| python3 -m pip cache purge 2>/dev/null || echo "No pip cache to clear" | |
| # Create Results directory in tutorial folder | |
| tutorial_results_dir="Tutorials/${{ matrix.tutorial.name }}/Results" | |
| mkdir -p "$tutorial_results_dir" | |
| echo "Created results directory: $tutorial_results_dir" | |
| # Set languages to test | |
| if [[ -n "${{ github.event.inputs.languages }}" ]]; then | |
| # Manual override via workflow_dispatch | |
| test_languages="${{ github.event.inputs.languages }}" | |
| echo "Using manually specified languages: $test_languages" | |
| else | |
| # Use languages detected for this specific tutorial | |
| test_languages="${{ matrix.tutorial.languages }}" | |
| echo "Using tutorial-specific languages: $test_languages" | |
| fi | |
| echo "TEST_LANGUAGES=$test_languages" >> $GITHUB_ENV | |
| echo "TUTORIAL_RESULTS_DIR=$tutorial_results_dir" >> $GITHUB_ENV | |
| echo "Will test languages: $test_languages" | |
| - name: Clean previous Files directory | |
| shell: bash | |
| run: | | |
| echo "Cleaning previous Files directory for: ${{ matrix.tutorial.name }}" | |
| files_dir="Tutorials/${{ matrix.tutorial.name }}/Files" | |
| if [[ -d "$files_dir" ]]; then | |
| echo "Found existing Files directory, removing..." | |
| rm -rf "$files_dir" | |
| echo "✅ Removed old Files directory" | |
| else | |
| echo "No existing Files directory found (this is fine for first run)" | |
| fi | |
| # Recreate empty Files directory | |
| mkdir -p "$files_dir" | |
| echo "✅ Created fresh Files directory" | |
| - name: Setup tutorial files in TutorialMaker | |
| shell: bash | |
| run: | | |
| echo "Setting up tutorial files for: ${{ matrix.tutorial.name }}" | |
| python3 Scripts/setup_tutorial_files.py \ | |
| "${{ matrix.tutorial.name }}" \ | |
| "Tutorials/${{ matrix.tutorial.name }}" \ | |
| --languages $TEST_LANGUAGES | |
| - name: Run tutorial tests | |
| run: | | |
| echo "Running tests for tutorial: ${{ matrix.tutorial.name }} (ID: ${{ matrix.tutorial.id }})" | |
| echo "Languages: $TEST_LANGUAGES" | |
| echo "Results directory: $TUTORIAL_RESULTS_DIR" | |
| # Check resources before running | |
| echo "=== Resources before test ===" | |
| free -h | |
| df -h / | |
| export DISPLAY=:99 | |
| # Change to repository root | |
| cd "$GITHUB_WORKSPACE" | |
| # Run the test script | |
| python3 Scripts/run_tutorial_tests_ci.py \ | |
| /opt/slicer/Slicer \ | |
| --tutorial "${{ matrix.tutorial.name }}" \ | |
| --languages $TEST_LANGUAGES \ | |
| --output "$TUTORIAL_RESULTS_DIR" \ | |
| --timeout 900 | |
| echo "Tutorial tests completed" | |
| # Check resources after running | |
| echo "=== Resources after test ===" | |
| free -h | |
| df -h / | |
| - name: Copy generated tutorial outputs | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "Copying generated tutorial outputs (HTML/MD)..." | |
| # Extract tutorial name without ID prefix | |
| tutorial_full_name="${{ matrix.tutorial.name }}" | |
| tutorial_name_only="${tutorial_full_name#*_}" | |
| echo "Full tutorial name: $tutorial_full_name" | |
| echo "Tutorial name (without ID): $tutorial_name_only" | |
| # Find TutorialMaker outputs directory | |
| # Try multiple possible locations | |
| for possible_dir in \ | |
| /opt/slicer/slicer.org/Extensions-*/TutorialMaker/lib/Slicer-*/qt-scripted-modules \ | |
| /opt/slicer/lib/Slicer-*/qt-scripted-modules/TutorialMaker \ | |
| /opt/slicer/lib/Slicer-*/extensions-*/TutorialMaker*; do | |
| if [[ -d "$possible_dir/Outputs" ]]; then | |
| echo "Found TutorialMaker outputs in: $possible_dir/Outputs" | |
| # List all outputs for debugging | |
| echo "Available output directories:" | |
| ls -la "$possible_dir/Outputs/" | grep "^d" || echo " (none)" | |
| echo "" | |
| # Copy outputs for this tutorial | |
| # Since TutorialMaker uses tutorial title (not filename) for folder names, | |
| # we search for any folder ending with the language code | |
| IFS=' ' read -ra LANG_ARRAY <<< "$TEST_LANGUAGES" | |
| for language in "${LANG_ARRAY[@]}"; do | |
| echo "Searching for outputs with language: $language" | |
| # Find all directories ending with _${language} | |
| found_output=false | |
| for output_dir in "$possible_dir/Outputs"/*_${language}; do | |
| if [[ -d "$output_dir" ]] && [[ ! "$output_dir" =~ __pycache__ ]]; then | |
| echo " ✅ Found: $output_dir" | |
| found_output=true | |
| # Create target directory in Files folder | |
| target_dir="Tutorials/$tutorial_full_name/Files/$language" | |
| mkdir -p "$target_dir" | |
| # Copy entire output directory (HTML, MD, images, etc.) | |
| echo " Copying all files..." | |
| cp -r "$output_dir"/* "$target_dir/" 2>/dev/null || true | |
| echo " ✅ Copied all files for $language" | |
| # List what was copied | |
| echo " Files copied:" | |
| ls -lh "$target_dir/" 2>/dev/null || echo " (empty)" | |
| # Only process the first match per language | |
| break | |
| fi | |
| done | |
| if [[ "$found_output" == "false" ]]; then | |
| echo " ⚠️ No output directory found ending with: _${language}" | |
| fi | |
| done | |
| # Clean up Outputs folder to prevent accumulation for next tutorial | |
| echo "" | |
| echo "🧹 Cleaning Outputs folder for next tutorial..." | |
| # Keep only essential folders (Raw, Annotations) and remove generated outputs | |
| for dir in "$possible_dir/Outputs"/*; do | |
| if [[ -d "$dir" ]]; then | |
| dir_name=$(basename "$dir") | |
| # Keep Raw, Annotations, __pycache__ folders | |
| if [[ "$dir_name" != "Raw" ]] && [[ "$dir_name" != "Annotations" ]] && [[ "$dir_name" != "__pycache__" ]]; then | |
| echo " Removing: $dir_name" | |
| rm -rf "$dir" 2>/dev/null || true | |
| fi | |
| fi | |
| done | |
| echo "✅ Cleanup completed" | |
| echo "Remaining directories:" | |
| ls -la "$possible_dir/Outputs/" | grep "^d" || echo " (none)" | |
| break | |
| fi | |
| done | |
| echo "" | |
| echo "Generated files summary:" | |
| if [[ -d "Tutorials/$tutorial_full_name/Files" ]]; then | |
| find "Tutorials/$tutorial_full_name/Files" -type f | |
| else | |
| echo " No files generated" | |
| fi | |
| - name: Collect Slicer logs | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "Collecting Slicer logs..." | |
| # Create directory for Slicer logs | |
| slicer_logs_dir="$TUTORIAL_RESULTS_DIR/slicer-logs" | |
| mkdir -p "$slicer_logs_dir" | |
| # Collect from common Slicer log locations on Linux | |
| log_locations=( | |
| "/tmp/Slicer*" | |
| "/tmp/slicer*" | |
| "$HOME/.config/NA-MIC/Slicer*.log" | |
| "$HOME/.config/NA-MIC/logs" | |
| "/var/tmp/Slicer*" | |
| ) | |
| # Get today's date for filtering | |
| today=$(date +%Y-%m-%d) | |
| for pattern in "${log_locations[@]}"; do | |
| for log_path in $pattern; do | |
| if [[ -e "$log_path" ]]; then | |
| echo "Found log location: $log_path" | |
| if [[ -d "$log_path" ]]; then | |
| # If it's a directory, copy recent log files | |
| echo " Copying recent logs from directory..." | |
| find "$log_path" -type f \( -name "*.log" -o -name "*.txt" -o -name "Slicer-*" \) 2>/dev/null | while IFS= read -r log_file; do | |
| # Check if file was modified today | |
| file_date=$(stat -c %y "$log_file" 2>/dev/null | cut -d' ' -f1) | |
| if [[ "$file_date" == "$today" ]] || [[ -z "$file_date" ]]; then | |
| if cp "$log_file" "$slicer_logs_dir/" 2>/dev/null; then | |
| echo " ✅ Copied: $(basename "$log_file")" | |
| fi | |
| fi | |
| done | |
| elif [[ -f "$log_path" ]]; then | |
| # If it's a file, check if it's from today | |
| file_date=$(stat -c %y "$log_path" 2>/dev/null | cut -d' ' -f1) | |
| if [[ "$file_date" == "$today" ]] || [[ -z "$file_date" ]]; then | |
| if cp "$log_path" "$slicer_logs_dir/" 2>/dev/null; then | |
| echo " ✅ Copied: $(basename "$log_path")" | |
| fi | |
| fi | |
| fi | |
| fi | |
| done | |
| done | |
| # Count collected files | |
| files_found=$(find "$slicer_logs_dir" -type f 2>/dev/null | wc -l) | |
| if [[ $files_found -eq 0 ]]; then | |
| echo "⚠️ No Slicer logs found" | |
| echo "no-logs-found" > "$slicer_logs_dir/README.txt" | |
| else | |
| echo "✅ Collected $files_found Slicer log files" | |
| echo "" | |
| echo "Log files collected:" | |
| ls -lh "$slicer_logs_dir/" | |
| fi | |
| - name: Process test results | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "Processing test results for: ${{ matrix.tutorial.name }}" | |
| echo "" | |
| if [[ -f "$TUTORIAL_RESULTS_DIR/test_report.json" ]]; then | |
| echo "✅ Test report found" | |
| cat "$TUTORIAL_RESULTS_DIR/test_report.json" | grep -E '"total_tests"|"successful_tests"|"failed_tests"|"success_rate"' || true | |
| # Show error hints if available | |
| error_hints=$(cat "$TUTORIAL_RESULTS_DIR/test_report.json" | grep -A 5 '"error_hints"' || true) | |
| if [[ -n "$error_hints" ]]; then | |
| echo "⚠️ Error hints: $error_hints" | |
| fi | |
| else | |
| echo "❌ No test report" | |
| # Show log files if available | |
| for log_file in "$TUTORIAL_RESULTS_DIR"/*.log; do | |
| if [[ -f "$log_file" ]]; then | |
| echo "Last 10 lines of $(basename "$log_file"):" | |
| tail -10 "$log_file" | |
| fi | |
| done | |
| fi | |
| - name: Verify generated files | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "=== Checking generated files ===" | |
| echo "Current directory: $(pwd)" | |
| echo "" | |
| # Check if we have files to commit | |
| has_results=false | |
| has_files=false | |
| if [[ -d "Tutorials/${{ matrix.tutorial.name }}/Results" ]]; then | |
| echo "✅ Results directory exists" | |
| ls -lh "Tutorials/${{ matrix.tutorial.name }}/Results/" | head -n 10 | |
| # Check for Slicer logs | |
| if [[ -d "Tutorials/${{ matrix.tutorial.name }}/Results/slicer-logs" ]]; then | |
| log_count=$(find "Tutorials/${{ matrix.tutorial.name }}/Results/slicer-logs" -type f | wc -l) | |
| echo "✅ Slicer logs: $log_count files found" | |
| fi | |
| has_results=true | |
| fi | |
| if [[ -d "Tutorials/${{ matrix.tutorial.name }}/Files" ]]; then | |
| echo "✅ Files directory exists" | |
| echo "Languages found:" | |
| ls -d "Tutorials/${{ matrix.tutorial.name }}/Files/"*/ 2>/dev/null || echo " No language folders" | |
| echo "" | |
| echo "Total files:" | |
| find "Tutorials/${{ matrix.tutorial.name }}/Files" -type f | wc -l | |
| has_files=true | |
| fi | |
| # Set environment variable for PR step | |
| if [[ "$has_results" == "true" ]] || [[ "$has_files" == "true" ]]; then | |
| echo "HAS_FILES=true" >> $GITHUB_ENV | |
| else | |
| echo "HAS_FILES=false" >> $GITHUB_ENV | |
| fi | |
| - name: Create Pull Request with test results | |
| if: always() && env.HAS_FILES == 'true' | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| committer: GitHub Action <action@github.com> | |
| author: GitHub Action <action@github.com> | |
| commit-message: | | |
| Update test results and generated outputs for ${{ matrix.tutorial.name }} [skip ci] | |
| - Languages tested: ${{ env.TEST_LANGUAGES }} | |
| - Tutorial ID: ${{ matrix.tutorial.id }} | |
| - Generated HTML/MD files included | |
| branch: test-results-${{ matrix.tutorial.id }}-${{ github.run_number }} | |
| delete-branch: true | |
| add-paths: | | |
| Tutorials/${{ matrix.tutorial.name }}/Results/ | |
| Tutorials/${{ matrix.tutorial.name }}/Files/ | |
| title: 'Test Results: ${{ matrix.tutorial.name }}' | |
| body: | | |
| ## 🧪 Tutorial Test Results | |
| **Tutorial:** ${{ matrix.tutorial.name }} (`${{ matrix.tutorial.id }}`) | |
| **Languages tested:** ${{ env.TEST_LANGUAGES }} | |
| ### Generated Content | |
| - ✅ Test results in `Results/` | |
| - ✅ Generated HTML/MD files in `Files/` (one folder per language) | |
| - ✅ Slicer execution logs in `Results/slicer-logs/` | |
| ### Files Included | |
| - Test reports (JSON) in `Results/` | |
| - Tutorial HTML files (localized) in `Files/{language}/` | |
| - Tutorial Markdown files (localized) in `Files/{language}/` | |
| - Slicer logs in `Results/slicer-logs/` | |
| --- | |
| *Auto-generated by [tutorial-tests.yml workflow](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})* | |
| labels: | | |
| tutorial-tests | |
| automated | |
| draft: false | |
| - name: List files to upload | |
| if: always() | |
| shell: bash | |
| run: | | |
| echo "=== Files prepared for upload ===" | |
| echo "" | |
| echo "Results directory:" | |
| if [[ -d "Tutorials/${{ matrix.tutorial.name }}/Results" ]]; then | |
| ls -lh "Tutorials/${{ matrix.tutorial.name }}/Results/" | |
| echo "" | |
| echo "Slicer logs:" | |
| if [[ -d "Tutorials/${{ matrix.tutorial.name }}/Results/slicer-logs" ]]; then | |
| ls -lh "Tutorials/${{ matrix.tutorial.name }}/Results/slicer-logs/" | |
| else | |
| echo " (no Slicer logs found)" | |
| fi | |
| else | |
| echo " (not found)" | |
| fi | |
| echo "" | |
| echo "Files directory (HTML/MD):" | |
| if [[ -d "Tutorials/${{ matrix.tutorial.name }}/Files" ]]; then | |
| find "Tutorials/${{ matrix.tutorial.name }}/Files" -type f -exec ls -lh {} \; | |
| else | |
| echo " (not found)" | |
| fi | |
| echo "" | |
| echo "Debug artifacts (Raw folder):" | |
| if [[ -d "debug_artifacts" ]]; then | |
| find "debug_artifacts" -type f | head -20 | |
| else | |
| echo " (not found)" | |
| fi | |
| - name: Upload test artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: test-results-${{ matrix.tutorial.id }} | |
| path: | | |
| Tutorials/${{ matrix.tutorial.name }}/Results/ | |
| Tutorials/${{ matrix.tutorial.name }}/Files/ | |
| debug_artifacts/ | |
| retention-days: 30 | |
| compression-level: 6 | |
| - name: Check test success | |
| if: always() | |
| shell: bash | |
| run: | | |
| if [[ -f "$TUTORIAL_RESULTS_DIR/test_report.json" ]]; then | |
| # Extract success rate using grep and basic text processing | |
| SUCCESS_RATE=$(grep '"success_rate"' "$TUTORIAL_RESULTS_DIR/test_report.json" | sed 's/.*: *\([0-9.]*\).*/\1/' || echo "0") | |
| echo "Tutorial ${{ matrix.tutorial.name }} success rate: $SUCCESS_RATE%" | |
| # Consider test successful if success rate >= 75% | |
| if (( $(echo "$SUCCESS_RATE >= 75.0" | bc -l 2>/dev/null) )); then | |
| echo "✅ Tutorial tests passed" | |
| else | |
| echo "❌ Tutorial tests failed - success rate too low" | |
| exit 1 | |
| fi | |
| else | |
| echo "❌ No test results available" | |
| exit 1 | |
| fi |