From 9f125156ad10e40a6c8bf300d40f9b1f5b1bf568 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 8 Jun 2022 15:16:35 -0400
Subject: [PATCH 01/24] add intel ci with build cache

---
 .github/workflows/intel_pr.yml | 47 ++++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 .github/workflows/intel_pr.yml

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
new file mode 100644
index 0000000000..41b9c14d39
--- /dev/null
+++ b/.github/workflows/intel_pr.yml
@@ -0,0 +1,47 @@
+on: [push, pull_request]
+jobs:
+  Code-Coverage:
+    runs-on: ubuntu-latest
+    container:
+      image: intel/oneapi-hpckit:devel-ubuntu20.04
+      env:
+        CC: mpiicc
+        FC: mpiifort
+        CFLAGS:  "-I/libs/include"
+        FCFLAGS: "-I/libs/include"
+        LDFLAGS: "-L/libs/lib"
+    steps:
+    - name: Cache dependencies
+      id: cache
+      uses: actions/cache@v3
+      with:
+        path: /libs
+        key: ${{ runner.os }}-intel-libs
+    - name: Install packages for building 
+      run: apt update && apt install -y autoconf libtool automake zlibc zlib1g-dev
+    - if: steps.cache.outputs.cache-hit != 'true'
+      name: Build netcdf
+      run: |
+        mkdir /libs
+        wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_12_2/source/hdf5-1.12.2.tar.gz
+        tar xf hdf5-1.12.2.tar.gz && cd hdf5-1.12.2
+        ./configure --prefix=/libs
+        make -j install && cd ..
+        wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.8.1.tar.gz
+        tar xf v4.8.1.tar.gz && cd netcdf-c-4.8.1
+        ./configure --prefix=/libs --enable-remote-fortran-bootstrap
+        make -j install
+        # sets this here to pass embeded configure checks(needs var from image)
+        export LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
+        make -j -k build-netcdf-fortran
+        make -j install-netcdf-fortran && cd
+    - name: checkout
+      uses: actions/checkout@v2
+    - name: Configure
+      run: autoreconf -if ./configure.ac && ./configure #--with-yaml
+    - name: Compile
+      run: make -j || make # parallel builds sometimes fail
+    - name: Run test suite
+      run: make -j check
+      env:
+        TEST_VERBOSE: 1

From 8f3c3a11aeb4477ed8075e8ecec0bdd6000fe58e Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 8 Jun 2022 15:52:31 -0400
Subject: [PATCH 02/24] move verbose flag, add ld lib path to check

---
 .github/workflows/intel_pr.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 41b9c14d39..d1e05c3004 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -10,6 +10,7 @@ jobs:
         CFLAGS:  "-I/libs/include"
         FCFLAGS: "-I/libs/include"
         LDFLAGS: "-L/libs/lib"
+        TEST_VERBOSE: 1
     steps:
     - name: Cache dependencies
       id: cache
@@ -42,6 +43,4 @@ jobs:
     - name: Compile
       run: make -j || make # parallel builds sometimes fail
     - name: Run test suite
-      run: make -j check
-      env:
-        TEST_VERBOSE: 1
+      run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"

From cca44ff33b976393d4f13efbb8b6f6f177b071fb Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Thu, 9 Jun 2022 14:36:59 -0400
Subject: [PATCH 03/24] set intel environment var for runtime faults

---
 .github/workflows/intel_pr.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index d1e05c3004..669ccd12a2 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -11,6 +11,7 @@ jobs:
         FCFLAGS: "-I/libs/include"
         LDFLAGS: "-L/libs/lib"
         TEST_VERBOSE: 1
+        I_MPI_FABRICS: "shm" # needed for mpi in image
     steps:
     - name: Cache dependencies
       id: cache

From c4e899d5a30eea1a2adb4d001417ab82a8cc11ec Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Tue, 14 Jun 2022 17:11:11 -0400
Subject: [PATCH 04/24] add a pe to failing test to check intel bug

---
 test_fms/diag_manager/test_diag_manager2.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_fms/diag_manager/test_diag_manager2.sh b/test_fms/diag_manager/test_diag_manager2.sh
index a4d36cf52b..8ae6ba21c3 100755
--- a/test_fms/diag_manager/test_diag_manager2.sh
+++ b/test_fms/diag_manager/test_diag_manager2.sh
@@ -457,7 +457,7 @@ test_diag_manager_23
 _EOF
 setup_test
 test_expect_success "Unstructured grid (test $my_test_count)" '
-  mpirun -n 1 ../test_diag_manager
+  mpirun -n 2 ../test_diag_manager
 '
 
 # test_diag_manager_time

From 7a3140331440c7da9737530bff3e534386b72b62 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Wed, 22 Jun 2022 15:14:33 -0400
Subject: [PATCH 05/24] add skips for shm issue

---
 .github/workflows/intel_pr.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 669ccd12a2..303669a455 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -43,5 +43,7 @@ jobs:
       run: autoreconf -if ./configure.ac && ./configure #--with-yaml
     - name: Compile
       run: make -j || make # parallel builds sometimes fail
+      env:
+        SKIP_TESTS: "test_mpp_update_domains.[1] test_update_domains_performance.[1]" # TODO oneapi fails on domain updates with one pe when using shm
     - name: Run test suite
       run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"

From d184ca1fc24448f87f8f726879f762632e61562f Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Wed, 22 Jun 2022 15:29:35 -0400
Subject: [PATCH 06/24] add libyaml

---
 .github/workflows/intel_pr.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 303669a455..09bad9fdb3 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -36,11 +36,15 @@ jobs:
         # sets this here to pass embeded configure checks(needs var from image)
         export LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
         make -j -k build-netcdf-fortran
-        make -j install-netcdf-fortran && cd
+        make -j install-netcdf-fortran
+        wget https://github.com/yaml/libyaml/releases/download/0.2.5/yaml-0.2.5.tar.gz
+        tar xf yaml-0.2.5.tar.gz && cd yaml-0.2.5
+        ./configure --prefix=/libs
+        make -j install && cd
     - name: checkout
       uses: actions/checkout@v2
     - name: Configure
-      run: autoreconf -if ./configure.ac && ./configure #--with-yaml
+      run: autoreconf -if ./configure.ac && ./configure --with-yaml
     - name: Compile
       run: make -j || make # parallel builds sometimes fail
       env:

From 63251fd11d6e46342c0e490d6c7a02991d6b5be3 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Thu, 23 Jun 2022 15:11:52 -0400
Subject: [PATCH 07/24] Fix skip flags

---
 .github/workflows/intel_pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 09bad9fdb3..ce7ed78f52 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -48,6 +48,6 @@ jobs:
     - name: Compile
       run: make -j || make # parallel builds sometimes fail
       env:
-        SKIP_TESTS: "test_mpp_update_domains.[1] test_update_domains_performance.[1]" # TODO oneapi fails on domain updates with one pe when using shm
+        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1" # TODO oneapi fails on domain updates with one pe when using shm
     - name: Run test suite
       run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"

From 22889b3bdcf52c13f550ac4d819e2c0e3b1eede8 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Thu, 23 Jun 2022 16:50:49 -0400
Subject: [PATCH 08/24] move skip var definition

---
 .github/workflows/intel_pr.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index ce7ed78f52..72230424dd 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -12,6 +12,7 @@ jobs:
         LDFLAGS: "-L/libs/lib"
         TEST_VERBOSE: 1
         I_MPI_FABRICS: "shm" # needed for mpi in image
+        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1" # TODO oneapi fails on domain updates with one pe when using shm
     steps:
     - name: Cache dependencies
       id: cache
@@ -47,7 +48,5 @@ jobs:
       run: autoreconf -if ./configure.ac && ./configure --with-yaml
     - name: Compile
       run: make -j || make # parallel builds sometimes fail
-      env:
-        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1" # TODO oneapi fails on domain updates with one pe when using shm
     - name: Run test suite
-      run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
+      run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH" SKIP_TESTS="$SKIP_TESTS"

From 19c818585de91d0d03eadd742175b5164fc74c4f Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Fri, 24 Jun 2022 16:44:46 -0400
Subject: [PATCH 09/24] Fix skips

---
 test_fms/test-lib.sh.in | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test_fms/test-lib.sh.in b/test_fms/test-lib.sh.in
index 2426297426..b983b48d84 100644
--- a/test_fms/test-lib.sh.in
+++ b/test_fms/test-lib.sh.in
@@ -232,13 +232,12 @@ match_pattern_list_ () {
     pInd=`expr index $arg '.'`
     tNameArg=`expr substr $arg 1 $pInd`
     tNamePattern=`expr substr $pattern_ 1 $pInd`
-    test "$tNameArg" != "$tNamePattern" && break
     isRange="`expr index $pattern_ '-'`"
     # individual test num
     if test $isRange -eq 0 ; then
       say "is range.."
       if test "$pattern_" = "$arg" ; then
-          return 0
+           test "$tNameArg" = "$tNamePattern" && return 0
       fi
     # num range
     else

From 24c37d3207c267348054f22d9bcddd9e9123d0d9 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Mon, 27 Jun 2022 12:46:49 -0400
Subject: [PATCH 10/24] change name, revert pe count change

---
 .github/workflows/intel_pr.yml              | 6 ++++--
 test_fms/diag_manager/test_diag_manager2.sh | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 72230424dd..416aff610d 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -1,6 +1,6 @@
 on: [push, pull_request]
 jobs:
-  Code-Coverage:
+  intel-autotools:
     runs-on: ubuntu-latest
     container:
       image: intel/oneapi-hpckit:devel-ubuntu20.04
@@ -12,7 +12,9 @@ jobs:
         LDFLAGS: "-L/libs/lib"
         TEST_VERBOSE: 1
         I_MPI_FABRICS: "shm" # needed for mpi in image
-        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1" # TODO oneapi fails on domain updates with one pe when using shm
+        # Skips tests with 1 pe domain updates
+        # intel bug causes failures with shm option(required by container)
+        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23"
     steps:
     - name: Cache dependencies
       id: cache
diff --git a/test_fms/diag_manager/test_diag_manager2.sh b/test_fms/diag_manager/test_diag_manager2.sh
index 8ae6ba21c3..a4d36cf52b 100755
--- a/test_fms/diag_manager/test_diag_manager2.sh
+++ b/test_fms/diag_manager/test_diag_manager2.sh
@@ -457,7 +457,7 @@ test_diag_manager_23
 _EOF
 setup_test
 test_expect_success "Unstructured grid (test $my_test_count)" '
-  mpirun -n 2 ../test_diag_manager
+  mpirun -n 1 ../test_diag_manager
 '
 
 # test_diag_manager_time

From 2e863276d03e34bd2c879c611eff8fdad3873e37 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Wed, 29 Jun 2022 12:05:36 -0400
Subject: [PATCH 11/24] revert debugging changes again

---
 .github/workflows/intel_pr.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 416aff610d..3008d5b51b 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -8,13 +8,12 @@ jobs:
         CC: mpiicc
         FC: mpiifort
         CFLAGS:  "-I/libs/include"
-        FCFLAGS: "-I/libs/include"
+        FCFLAGS: "-I/libs/include -g -traceback"
         LDFLAGS: "-L/libs/lib"
         TEST_VERBOSE: 1
         I_MPI_FABRICS: "shm" # needed for mpi in image
-        # Skips tests with 1 pe domain updates
-        # intel bug causes failures with shm option(required by container)
-        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23"
+        # intel bug causes some failures with shm option(required in container)
+        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23 test_mpp_io2.1 test_io_R4_R8.1"
     steps:
     - name: Cache dependencies
       id: cache
@@ -25,7 +24,7 @@ jobs:
     - name: Install packages for building 
       run: apt update && apt install -y autoconf libtool automake zlibc zlib1g-dev
     - if: steps.cache.outputs.cache-hit != 'true'
-      name: Build netcdf
+    - name: Build netcdf
       run: |
         mkdir /libs
         wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_12_2/source/hdf5-1.12.2.tar.gz
@@ -36,7 +35,7 @@ jobs:
         tar xf v4.8.1.tar.gz && cd netcdf-c-4.8.1
         ./configure --prefix=/libs --enable-remote-fortran-bootstrap
         make -j install
-        # sets this here to pass embeded configure checks(needs var from image)
+        # sets this here to pass embeded configure checks
         export LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
         make -j -k build-netcdf-fortran
         make -j install-netcdf-fortran
@@ -49,6 +48,6 @@ jobs:
     - name: Configure
       run: autoreconf -if ./configure.ac && ./configure --with-yaml
     - name: Compile
-      run: make -j || make # parallel builds sometimes fail
+      run: make -j || make
     - name: Run test suite
-      run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH" SKIP_TESTS="$SKIP_TESTS"
+      run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH" SKIP_TESTS="$SKIP_TESTS" VERBOSE=1

From 5de8f1ec6e1f4185b0ad33972ad5e6a53a62f195 Mon Sep 17 00:00:00 2001
From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com>
Date: Wed, 29 Jun 2022 12:08:40 -0400
Subject: [PATCH 12/24] remove extra dash

---
 .github/workflows/intel_pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 3008d5b51b..152f85d777 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -24,7 +24,7 @@ jobs:
     - name: Install packages for building 
       run: apt update && apt install -y autoconf libtool automake zlibc zlib1g-dev
     - if: steps.cache.outputs.cache-hit != 'true'
-    - name: Build netcdf
+      name: Build netcdf
       run: |
         mkdir /libs
         wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_12_2/source/hdf5-1.12.2.tar.gz

From 133a0b6644e2cf124bc901c26ce477bbca3b86f9 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Wed, 29 Jun 2022 14:40:11 -0400
Subject: [PATCH 13/24] fix mpp_io failures

---
 .github/workflows/intel_pr.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index 152f85d777..bdf2a0b887 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -13,7 +13,7 @@ jobs:
         TEST_VERBOSE: 1
         I_MPI_FABRICS: "shm" # needed for mpi in image
         # intel bug causes some failures with shm option(required in container)
-        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23 test_mpp_io2.1 test_io_R4_R8.1"
+        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23"
     steps:
     - name: Cache dependencies
       id: cache
@@ -50,4 +50,4 @@ jobs:
     - name: Compile
       run: make -j || make
     - name: Run test suite
-      run: make -j check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH" SKIP_TESTS="$SKIP_TESTS" VERBOSE=1
+      run: make check LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH" SKIP_TESTS="$SKIP_TESTS" VERBOSE=1

From 799a68dc61b84344578bf0c7c5b782661130cc93 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscgpu50-d.gfdl.noaa.gov>
Date: Thu, 30 Jun 2022 09:50:41 -0400
Subject: [PATCH 14/24] Remove parallworks ci, push trigger and change image
 tag to a specific versions

---
 .github/workflows/intel_pr.yml               |  6 ++--
 .github/workflows/parallelWorks_intel_pr.yml | 37 --------------------
 2 files changed, 3 insertions(+), 40 deletions(-)
 delete mode 100644 .github/workflows/parallelWorks_intel_pr.yml

diff --git a/.github/workflows/intel_pr.yml b/.github/workflows/intel_pr.yml
index bdf2a0b887..d95519fbf2 100644
--- a/.github/workflows/intel_pr.yml
+++ b/.github/workflows/intel_pr.yml
@@ -1,9 +1,9 @@
-on: [push, pull_request]
+on: pull_request
 jobs:
   intel-autotools:
     runs-on: ubuntu-latest
     container:
-      image: intel/oneapi-hpckit:devel-ubuntu20.04
+      image: intel/oneapi-hpckit:2022.2-devel-ubuntu20.04
       env:
         CC: mpiicc
         FC: mpiifort
@@ -21,7 +21,7 @@ jobs:
       with:
         path: /libs
         key: ${{ runner.os }}-intel-libs
-    - name: Install packages for building 
+    - name: Install packages for building
       run: apt update && apt install -y autoconf libtool automake zlibc zlib1g-dev
     - if: steps.cache.outputs.cache-hit != 'true'
       name: Build netcdf
diff --git a/.github/workflows/parallelWorks_intel_pr.yml b/.github/workflows/parallelWorks_intel_pr.yml
deleted file mode 100644
index cc8af5cfc8..0000000000
--- a/.github/workflows/parallelWorks_intel_pr.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-name: Pull Request CI libFMS with intel18 and intel21
-
-on: [pull_request,workflow_dispatch]
-jobs:
-   parallelWorks:
-    runs-on: [self-hosted, pw-platform]
-    strategy:
-      fail-fast: false
-      max-parallel: 2
-      matrix:
-        include:
-# Turn this back on when fixed
-                - runname: FMS with intel 18
-                  runscript: python3 /home/Thomas.Robinson/pw/storage/pw_api_python/PRFMSintel18StartClusters.py $GITHUB_REF
-# Runs on FMS_CONTAINER_CI cluster
-                - runname: FMS with intel 2021 container
-                  runscript: python3 /home/Thomas.Robinson/pw/storage/pw_api_python/PRFMSintel21StartClusters.py $GITHUB_REF
-    steps:
-                - name: FMS make check on paralellWorks
-                  env:
-                          RUNNAME: ${{ matrix.runname }}
-                          RUNSCRIPT: ${{ matrix.runscript }}
-                  run: $RUNSCRIPT
-   ShutDownCluster:
-    runs-on: [self-hosted, pw-platform]
-    if: always()
-    needs: [parallelWorks]
-    strategy:
-      matrix:
-        include:
-                - cluster: FMS_CONTAINER_CI
-                - cluster: fms_intel18_ci
-    steps:
-                - name: Turn off cluster
-                  env:
-                          CLUSTER: ${{ matrix.cluster }}
-                  run: python3 /home/Thomas.Robinson/pw/storage/pw_api_python/stopClusters.py $CLUSTER

From 51e719678a99dde168d3198a60256f75128f75f9 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 6 Jul 2022 10:30:12 -0400
Subject: [PATCH 15/24] CI output for coverage script

---
 test_fms/intel_coverage.sh.in | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/test_fms/intel_coverage.sh.in b/test_fms/intel_coverage.sh.in
index bc452277be..92ea93257f 100755
--- a/test_fms/intel_coverage.sh.in
+++ b/test_fms/intel_coverage.sh.in
@@ -71,19 +71,41 @@ cat <<_EOF > coverage-report/fms-coverage.html
 </head>
 <body>
 _EOF
-
+# comment that seems to prevent the _EOF from inexplicably being included in the output file
+total=0
+covered=0
 # generate code coverage report and links for each (src) subdirectory
-for dir in `find . -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | grep -v test_fms | sort`
+# also sums the covered and total block counts
+for dir in `find . -not -path '*/.*' -maxdepth 1 -mindepth 1 -type d -printf '%f\n' | grep -v test_fms | grep -v 'coverage-*'`
 do
   cd $dir
+  echo $dir
   if [ ! -z "`find . -type f -name "*.spi"`" ] ; then
     mkdir ../coverage-report/$dir
     codecov -prj $dir -spi pgopti.spi -dpi ../fms-global-coverage.dpi
+
+    tmp="`sed -n 54p CodeCoverage/__CODE_COVERAGE.HTML | sed 's/.*>\(.*\)<.*/\1/' | sed 's/,//' `"
+    total="`expr $tmp + $total`"
+    tmp="`sed -n 55p CodeCoverage/__CODE_COVERAGE.HTML | sed 's/.*>\(.*\)<.*/\1/' | sed 's/,//' `"
+    covered="`expr $tmp + $covered`"
+
     mv CODE_COVERAGE.HTML ../coverage-report/$dir
     mv CodeCoverage ../coverage-report/$dir
     echo "<p><a href=\"${dir}/CODE_COVERAGE.HTML\">${dir}</a></p>" >> ../coverage-report/fms-coverage.html
   fi
   cd ..
 done
+percentage="$(echo "scale=2; ${covered}*100/${total}" | bc)"
+
 echo "</body>" >> coverage-report/fms-coverage.html
 echo "</html>" >> coverage-report/fms-coverage.html
+# print results for total coverage since each report is per directory
+# extra output if in CI
+echo "$covered covered functions out of $total total (${percentage}%)"
+if [ "$CI" ]; then
+  echo "::set-output name=total::${total}"
+  echo "::set-output name=covered::${covered}"
+  echo "::set-output name=percentage::${percentage}"
+  echo "::notice title=Coverage Info::${percentage}% function coverage. ${covered} covered functions out of ${total} total"
+  echo "${percentage}" > /coverage-percent.txt
+fi

From af8de91810e71453d6fdb41f34d7b9c987d0a689 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 6 Jul 2022 10:53:44 -0400
Subject: [PATCH 16/24] Add coverage action

---
 .github/workflows/intel_coverage.yml | 78 ++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 .github/workflows/intel_coverage.yml

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
new file mode 100644
index 0000000000..914d980760
--- /dev/null
+++ b/.github/workflows/intel_coverage.yml
@@ -0,0 +1,78 @@
+on: [push, pull_request]
+jobs:
+  intel-autotools:
+    runs-on: ubuntu-latest
+    container:
+      image: intel/oneapi-hpckit:2022.2-devel-ubuntu20.04
+      env:
+        CC: mpiicc
+        FC: mpiifort
+        CFLAGS:  "-I/libs/include"
+        FCFLAGS: "-I/libs/include -g -traceback"
+        LDFLAGS: "-L/libs/lib"
+        TEST_VERBOSE: 1
+        I_MPI_FABRICS: "shm" # needed for mpi in image
+        # intel bug causes some failures with shm option(required in container)
+        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23"
+    steps:
+    - name: Cache dependencies
+      id: cache
+      uses: actions/cache@v3
+      with:
+        path: /libs
+        key: ${{ runner.os }}-intel-libs
+    - name: Install packages for building
+      run: apt update && apt install -y autoconf libtool automake zlibc zlib1g-dev
+    - if: steps.cache.outputs.cache-hit != 'true'
+      name: Build netcdf
+      run: |
+        mkdir /libs
+        wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_12_2/source/hdf5-1.12.2.tar.gz
+        tar xf hdf5-1.12.2.tar.gz && cd hdf5-1.12.2
+        ./configure --prefix=/libs
+        make -j install && cd ..
+        wget https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.8.1.tar.gz
+        tar xf v4.8.1.tar.gz && cd netcdf-c-4.8.1
+        ./configure --prefix=/libs --enable-remote-fortran-bootstrap
+        make -j install
+        # sets this here to pass embeded configure checks
+        export LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
+        make -j -k build-netcdf-fortran
+        make -j install-netcdf-fortran
+        wget https://github.com/yaml/libyaml/releases/download/0.2.5/yaml-0.2.5.tar.gz
+        tar xf yaml-0.2.5.tar.gz && cd yaml-0.2.5
+        ./configure --prefix=/libs
+        make -j install && cd
+    - name: checkout
+      uses: actions/checkout@v2
+    - name: Configure
+      run: autoreconf -if ./configure.ac && ./configure --with-yaml
+    - name: Compile
+      run: make -j || make
+    - name: Run test suite
+      run: make check-code-coverage LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
+    - name: Archive code coverage results
+      uses: actions/upload-artifact@v3
+      with:
+        name: code-coverage-report
+        path: /coverage-percent.txt
+    # get coverage results from last successful main run for comparison
+    - name: Download last results
+      uses: dawidd6/action-download-artifact@v2
+      continue-on-error: true
+      with:
+        workflow: coverage.yml
+        branch: coverage-action
+        workflow_conclusion: completed
+    - name: Compare coverage
+      run: |
+        if [ -f code-coverage-report/coverage-percent.txt ]; then
+          archive_val="`cat code-coverage-report/coverage-percent.txt`"
+          if [ "`expr $archive_val '>' $RESULT`" -eq 1 ]; then
+            echo "::warning title=Coverage Dropped:: Function coverage has decreased between runs from $archive_val to $RESULT"
+          fi
+        else
+          echo "::warning title=Coverage Not Checked:: Archived results could not be downloaded"
+        fi
+      env:
+        RESULT: ${{ steps.report.outputs.percentage }}

From c4aa57ed346d50b712e6b36e5fb4d459162c1f0e Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 6 Jul 2022 12:21:39 -0400
Subject: [PATCH 17/24] Add (and comment out) conditionals for checking against
 reports run on main and small fixes to run

---
 .github/workflows/intel_coverage.yml | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index 914d980760..acd92592e2 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -1,6 +1,14 @@
-on: [push, pull_request]
+# Ryan Mulhall 7/22
+# Uses an intel image to compile,test and generate a code coverage report.
+# If triggered by PR, it will compare coverage with the last main archived result
+# and output a warning for decreases in function coverage.
+on:
+  push:
+    branches:
+      - main
+  pull_request:
 jobs:
-  intel-autotools:
+  generate-coverage-report:
     runs-on: ubuntu-latest
     container:
       image: intel/oneapi-hpckit:2022.2-devel-ubuntu20.04
@@ -46,25 +54,31 @@ jobs:
     - name: checkout
       uses: actions/checkout@v2
     - name: Configure
-      run: autoreconf -if ./configure.ac && ./configure --with-yaml
+      run: autoreconf -if ./configure.ac && ./configure --enable-code-coverage --with-yaml
     - name: Compile
       run: make -j || make
     - name: Run test suite
+      id: report
       run: make check-code-coverage LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
+    #- if: ${{ github.ref == 'ref/heads/main' }}
     - name: Archive code coverage results
       uses: actions/upload-artifact@v3
       with:
         name: code-coverage-report
         path: /coverage-percent.txt
     # get coverage results from last successful main run for comparison
+    # the steps below only run on pull requests
+    #- if: ${{ github.ref != 'refs/heads/main' }}
     - name: Download last results
       uses: dawidd6/action-download-artifact@v2
       continue-on-error: true
       with:
-        workflow: coverage.yml
-        branch: coverage-action
+        workflow: intel_coverage.yml
+        branch: intel-coverage-action # TODO will be main when in repo
         workflow_conclusion: completed
+    #- if: ${{ github.ref != 'refs/heads/main' }}
     - name: Compare coverage
+      continue-on-error: true
       run: |
         if [ -f code-coverage-report/coverage-percent.txt ]; then
           archive_val="`cat code-coverage-report/coverage-percent.txt`"

From 14c3b7edbdd8154488a49efb7fe631b4f584833b Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 6 Jul 2022 12:26:15 -0400
Subject: [PATCH 18/24] revert triggers for now

---
 .github/workflows/intel_coverage.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index acd92592e2..05d25acd9f 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -2,11 +2,12 @@
 # Uses an intel image to compile,test and generate a code coverage report.
 # If triggered by PR, it will compare coverage with the last main archived result
 # and output a warning for decreases in function coverage.
-on:
-  push:
-    branches:
-      - main
-  pull_request:
+# on:
+#  push:
+#    branches:
+#      - main
+#  pull_request:
+on: [push, pull_request]
 jobs:
   generate-coverage-report:
     runs-on: ubuntu-latest

From d0936816c88c5e34041d8920aa0567e8d0252fa7 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 6 Jul 2022 14:40:50 -0400
Subject: [PATCH 19/24] fix percentage calc, add error message

---
 .github/workflows/intel_coverage.yml | 6 +++---
 test_fms/intel_coverage.sh.in        | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index 05d25acd9f..b0380c8f75 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -31,9 +31,9 @@ jobs:
         path: /libs
         key: ${{ runner.os }}-intel-libs
     - name: Install packages for building
-      run: apt update && apt install -y autoconf libtool automake zlibc zlib1g-dev
+      run: apt update && apt install -y autoconf libtool automake zlibc zlib1g-dev bc
     - if: steps.cache.outputs.cache-hit != 'true'
-      name: Build netcdf
+      name: Build dependencies
       run: |
         mkdir /libs
         wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_12_2/source/hdf5-1.12.2.tar.gz
@@ -84,7 +84,7 @@ jobs:
         if [ -f code-coverage-report/coverage-percent.txt ]; then
           archive_val="`cat code-coverage-report/coverage-percent.txt`"
           if [ "`expr $archive_val '>' $RESULT`" -eq 1 ]; then
-            echo "::warning title=Coverage Dropped:: Function coverage has decreased between runs from $archive_val to $RESULT"
+            echo "::warning title=Coverage Dropped:: Function/subroutine coverage has decreased between runs from $archive_val to $RESULT"
           fi
         else
           echo "::warning title=Coverage Not Checked:: Archived results could not be downloaded"
diff --git a/test_fms/intel_coverage.sh.in b/test_fms/intel_coverage.sh.in
index 92ea93257f..fcf9284153 100755
--- a/test_fms/intel_coverage.sh.in
+++ b/test_fms/intel_coverage.sh.in
@@ -106,6 +106,7 @@ if [ "$CI" ]; then
   echo "::set-output name=total::${total}"
   echo "::set-output name=covered::${covered}"
   echo "::set-output name=percentage::${percentage}"
-  echo "::notice title=Coverage Info::${percentage}% function coverage. ${covered} covered functions out of ${total} total"
+  test -z "${percentage}" && echo "::error title=Percentage calculation failed::" && exit 1
+  echo "::notice title=Coverage Info::${percentage}% of routines/functions coverage. ${covered} covered functions out of ${total} total"
   echo "${percentage}" > /coverage-percent.txt
 fi

From 2c91d17ca613503acf1b25eb9dc5cf7da7ff2c4d Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Mon, 11 Jul 2022 09:02:23 -0400
Subject: [PATCH 20/24] fix percentage comparison pt. 2

---
 .github/workflows/intel_coverage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index b0380c8f75..02112cb94c 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -83,7 +83,7 @@ jobs:
       run: |
         if [ -f code-coverage-report/coverage-percent.txt ]; then
           archive_val="`cat code-coverage-report/coverage-percent.txt`"
-          if [ "`expr $archive_val '>' $RESULT`" -eq 1 ]; then
+          if [ 1 -eq "$(echo "${archive_val} > ${RESULT}" | bc)" ]; then
             echo "::warning title=Coverage Dropped:: Function/subroutine coverage has decreased between runs from $archive_val to $RESULT"
           fi
         else

From 49a9439efef6d69826a7d5c83fbd94c166220018 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Tue, 12 Jul 2022 11:56:03 -0400
Subject: [PATCH 21/24] test warning message

---
 .github/workflows/intel_coverage.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index 02112cb94c..5195114fbf 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -22,7 +22,7 @@ jobs:
         TEST_VERBOSE: 1
         I_MPI_FABRICS: "shm" # needed for mpi in image
         # intel bug causes some failures with shm option(required in container)
-        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23"
+        SKIP_TESTS: "test_mpp_update_domains.1 test_update_domains_performance.1 test_diag_manager2.23 test_field_manager2.1"
     steps:
     - name: Cache dependencies
       id: cache
@@ -84,7 +84,7 @@ jobs:
         if [ -f code-coverage-report/coverage-percent.txt ]; then
           archive_val="`cat code-coverage-report/coverage-percent.txt`"
           if [ 1 -eq "$(echo "${archive_val} > ${RESULT}" | bc)" ]; then
-            echo "::warning title=Coverage Dropped:: Function/subroutine coverage has decreased between runs from $archive_val to $RESULT"
+            echo "::error title=Coverage Dropped:: Function/subroutine coverage has decreased between runs from $archive_val to $RESULT"
           fi
         else
           echo "::warning title=Coverage Not Checked:: Archived results could not be downloaded"

From 16c525fa30bac40fd68ca6fa4b9ebc96211c648e Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Wed, 13 Jul 2022 15:55:34 -0400
Subject: [PATCH 22/24] switch to an actual error on coverage loss, and test
 comparisons

---
 .github/workflows/intel_coverage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index 5195114fbf..adacffb643 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -79,12 +79,12 @@ jobs:
         workflow_conclusion: completed
     #- if: ${{ github.ref != 'refs/heads/main' }}
     - name: Compare coverage
-      continue-on-error: true
       run: |
         if [ -f code-coverage-report/coverage-percent.txt ]; then
           archive_val="`cat code-coverage-report/coverage-percent.txt`"
           if [ 1 -eq "$(echo "${archive_val} > ${RESULT}" | bc)" ]; then
             echo "::error title=Coverage Dropped:: Function/subroutine coverage has decreased between runs from $archive_val to $RESULT"
+            exit 1
           fi
         else
           echo "::warning title=Coverage Not Checked:: Archived results could not be downloaded"

From 2dfa37c97b0b1feab301d95bc57c7f3fdc3e6df7 Mon Sep 17 00:00:00 2001
From: rem1776 <Ryan.Mulhall@lscamd50-d.gfdl.noaa.gov>
Date: Fri, 15 Jul 2022 12:00:17 -0400
Subject: [PATCH 23/24] last few changes to test with main

---
 .github/workflows/intel_coverage.yml | 40 ++++++++++++++--------------
 test_fms/intel_coverage.sh.in        |  2 +-
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index adacffb643..9aea8e343c 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -2,12 +2,12 @@
 # Uses an intel image to compile,test and generate a code coverage report.
 # If triggered by PR, it will compare coverage with the last main archived result
 # and output a warning for decreases in function coverage.
-# on:
-#  push:
-#    branches:
-#      - main
-#  pull_request:
-on: [push, pull_request]
+#on: [push, pull_request]
+ on:
+  push:
+    branches:
+      - main
+  pull_request:
 jobs:
   generate-coverage-report:
     runs-on: ubuntu-latest
@@ -61,29 +61,23 @@ jobs:
     - name: Run test suite
       id: report
       run: make check-code-coverage LD_LIBRARY_PATH="/libs/lib:$LD_LIBRARY_PATH"
-    #- if: ${{ github.ref == 'ref/heads/main' }}
-    - name: Archive code coverage results
-      uses: actions/upload-artifact@v3
-      with:
-        name: code-coverage-report
-        path: /coverage-percent.txt
     # get coverage results from last successful main run for comparison
     # the steps below only run on pull requests
-    #- if: ${{ github.ref != 'refs/heads/main' }}
-    - name: Download last results
+    - if: ${{ github.ref != 'refs/heads/main' }}
+      name: Download last results
       uses: dawidd6/action-download-artifact@v2
       continue-on-error: true
       with:
         workflow: intel_coverage.yml
-        branch: intel-coverage-action # TODO will be main when in repo
-        workflow_conclusion: completed
-    #- if: ${{ github.ref != 'refs/heads/main' }}
-    - name: Compare coverage
+        branch: main
+        workflow_conclusion: success
+    - if: ${{ github.ref != 'refs/heads/main' }}
+      name: Compare coverage
       run: |
         if [ -f code-coverage-report/coverage-percent.txt ]; then
           archive_val="`cat code-coverage-report/coverage-percent.txt`"
-          if [ 1 -eq "$(echo "${archive_val} > ${RESULT}" | bc)" ]; then
-            echo "::error title=Coverage Dropped:: Function/subroutine coverage has decreased between runs from $archive_val to $RESULT"
+          if [ 1 -eq "$(echo "${archive_val} > ${RESULT}" | bc)" ]; the
+            echo "::error title=Coverage Dropped:: This pull request will decrease coverage from $archive_val to $RESULT"
             exit 1
           fi
         else
@@ -91,3 +85,9 @@ jobs:
         fi
       env:
         RESULT: ${{ steps.report.outputs.percentage }}
+    - if: ${{ github.ref == 'ref/heads/main' }}
+      name: Archive code coverage results
+      uses: actions/upload-artifact@v3
+      with:
+        name: code-coverage-report
+        path: /coverage-percent.txt
diff --git a/test_fms/intel_coverage.sh.in b/test_fms/intel_coverage.sh.in
index fcf9284153..87383089fb 100755
--- a/test_fms/intel_coverage.sh.in
+++ b/test_fms/intel_coverage.sh.in
@@ -107,6 +107,6 @@ if [ "$CI" ]; then
   echo "::set-output name=covered::${covered}"
   echo "::set-output name=percentage::${percentage}"
   test -z "${percentage}" && echo "::error title=Percentage calculation failed::" && exit 1
-  echo "::notice title=Coverage Info::${percentage}% of routines/functions coverage. ${covered} covered functions out of ${total} total"
+  echo "::notice title=Coverage Info::${percentage}% of routine/function coverage. ${covered} covered functions out of ${total} total"
   echo "${percentage}" > /coverage-percent.txt
 fi

From fff93a81ad31b3a704c4a595ef2aea1e75b32bde Mon Sep 17 00:00:00 2001
From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com>
Date: Fri, 15 Jul 2022 12:19:27 -0400
Subject: [PATCH 24/24] fix indent

---
 .github/workflows/intel_coverage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/intel_coverage.yml b/.github/workflows/intel_coverage.yml
index 9aea8e343c..afc499d751 100644
--- a/.github/workflows/intel_coverage.yml
+++ b/.github/workflows/intel_coverage.yml
@@ -3,7 +3,7 @@
 # If triggered by PR, it will compare coverage with the last main archived result
 # and output a warning for decreases in function coverage.
 #on: [push, pull_request]
- on:
+on:
   push:
     branches:
       - main