Ensure unit tests don't fail silently

.github/workflows/regression.yml

@@ -0,0 +1,116 @@
+name: legolas
+
+on:
+  push:
+    branches: [master, develop]
+  pull_request:
+    branches: [master, develop]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        name: [regression]
+        os: [ubuntu-latest]
+        compiler: [gfortran-11]
+        coverage: [true]
+
+    env:
+      FC: ${{ matrix.compiler }}
+      LEGOLASDIR: /home/runner/work/legolas/legolas
+      ARPACK_ROOT: /home/runner/work/legolas/legolas/tests/arpack-ng
+
+    name: ${{ matrix.name }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.9
+
+    - name: Install cmake
+      uses: jwlawson/actions-setup-cmake@v1.13.0
+      with:
+        cmake-version: "3.24.x"
+
+    - name: Install Legolas dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install ${{ matrix.compiler }}
+        sudo apt-get install libblas-dev
+        sudo apt-get install liblapack-dev
+        sudo apt-get install lcov
+        ${FC} --version
+        cmake --version
+        gcov --version
+
+    - name: Install Python dependencies & Pylbo
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest numpy matplotlib f90nml tqdm psutil pytest-mpl
+        cd post_processing
+        python setup.py develop
+
+    - name: Cache ARPACK
+      id: arpack-cache
+      uses: actions/cache@v1
+      with:
+        path: tests/arpack-ng/
+        key: ${{ runner.os }}-arpackv3
+
+    - name: Build ARPACK
+      if: steps.arpack-cache.outputs.cache-hit != 'true'
+      run: |
+        cd tests
+        git clone https://github.com/opencollab/arpack-ng.git
+        cd arpack-ng
+        mkdir build
+        mkdir installed
+        cd build
+        cmake -DEXAMPLES=OFF -DMPI=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=../installed ..
+        make -j 2
+        sudo make -j 2 install
+
+    - name: Compile Legolas
+      run: |
+        mkdir build
+        cd build
+        if [[ "${{ matrix.coverage }}" ]]; then
+          cmake -DCoverage=ON ..
+        else
+          cmake -DDebug=ON ..
+        fi
+        make -j 2
+
+    - name: Run tests
+      run: |
+        cd $LEGOLASDIR/tests/regression_tests
+        pytest
+
+    - name: Generate coverage report
+      if: ${{ matrix.coverage }}
+      run: |
+        mkdir coverage
+        cd coverage
+        lcov --capture --directory $LEGOLASDIR/build \
+            --output-file ${{ matrix.name }}.info \
+            --gcov-tool /usr/bin/gcov-11
+        # filter out coverage files
+        find $LEGOLASDIR/build -name '*.gc*' -delete
+
+    - name: Archive failed logs
+      uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: failed_logs
+        path: tests/regression_tests/test_results
+
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v1
+      if: ${{ matrix.coverage }}
+      with:
+        files: ./coverage/${{ matrix.name }}.info
+        flags: legolas

.github/workflows/legolas.yml → .github/workflows/unit.yml

@@ -13,18 +13,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: [unit-tests, regression]
         os: [ubuntu-latest]
-        compiler: [gfortran-11]
-        coverage: [true]
+        compiler: [gfortran-9, gfortran-10, gfortran-12]
+        coverage: [false]
+        include:
+          - os: ubuntu-latest
+            compiler: gfortran-11
+            coverage: true
 
     env:
       FC: ${{ matrix.compiler }}
       PFUNIT_DIR: /home/runner/work/legolas/legolas/tests/pFUnit/build/installed
       LEGOLASDIR: /home/runner/work/legolas/legolas
       ARPACK_ROOT: /home/runner/work/legolas/legolas/tests/arpack-ng
 
-    name: ${{ matrix.name }}
+    name: unit-tests / ${{ matrix.os }} / ${{ matrix.compiler }}
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.9
@@ -60,7 +63,7 @@ jobs:
       uses: actions/cache@v1
       with:
         path: tests/pFUnit/
-        key: ${{ runner.os }}-pfunitv2
+        key: ${{ runner.os }}-${{ matrix.compiler }}-pfunitv1
 
     - name: Build pFUnit
       if: steps.pfunit-cache.outputs.cache-hit != 'true'
@@ -79,7 +82,7 @@ jobs:
       uses: actions/cache@v1
       with:
         path: tests/arpack-ng/
-        key: ${{ runner.os }}-arpackv3
+        key: ${{ runner.os }}-${{ matrix.compiler }}-arpackv1
 
     - name: Build ARPACK
       if: steps.arpack-cache.outputs.cache-hit != 'true'
@@ -107,17 +110,16 @@ jobs:
 
     - name: Run tests
       run: |
-        if [[ "${{ matrix.name }}" == "unit-tests" ]]; then
-          cd $LEGOLASDIR/tests/unit_tests
-          mkdir build
-          cd build
-          cmake -DCoverage=ON ..
-          make -j 2
-          cd ..
-          ./test_legolas
-        elif [[ "${{ matrix.name }}" == "regression" ]]; then
-          cd $LEGOLASDIR/tests/regression_tests
-          pytest
+        cd $LEGOLASDIR/tests/unit_tests
+        mkdir build
+        cd build
+        cmake -DCoverage=ON ..
+        make -j 2
+        cd ..
+        # ensure failures are caught, pFUnit sometimes silently fails?
+        ./test_legolas | tee test_legolas.log
+        if grep -q "FAILURES" test_legolas.log; then
+          exit 1
         fi
 
     - name: Generate coverage report
@@ -126,7 +128,7 @@ jobs:
         mkdir coverage
         cd coverage
         lcov --capture --directory $LEGOLASDIR/build \
-            --output-file ${{ matrix.name }}.info \
+            --output-file unit-tests-${{ matrix.compiler }}.info \
             --gcov-tool /usr/bin/gcov-11
         # filter out coverage files
         find $LEGOLASDIR/build -name '*.gc*' -delete
@@ -142,5 +144,5 @@ jobs:
       uses: codecov/codecov-action@v1
       if: ${{ matrix.coverage }}
       with:
-        files: ./coverage/${{ matrix.name }}.info
+        files: ./coverage/unit-tests-${{ matrix.compiler }}.info
         flags: legolas

src/solvers/arnoldi/mod_arpack_type.f08

@@ -172,13 +172,15 @@ subroutine set_nev(this, nev)
       call logger%error( &
         "Arnoldi: number of eigenvalues must be >= 0 but got " // str(nev) &
       )
+      this%nev = 0
       return
     end if
     if (nev >= this%evpdim) then
       call logger%error( &
         "Arnoldi: number of eigenvalues (" // str(nev) &
         // ") >= " // "matrix size (" // str(this%evpdim) // ")" &
       )
+      this%nev = 0
       return
     end if
     this%nev = nev

tests/unit_tests/mod_test_input.pf

@@ -54,7 +54,9 @@ contains
     call read_parfile("test_parfiles/parfile_dryrun.par", settings)
     @assertEqual("none", settings%solvers%get_solver())
     @assertFalse(settings%io%write_eigenfunctions)
-    @assertFalse(settings%io%write_matrices)
+    @assertFalse(settings%io%write_derived_eigenfunctions)
+    @assertFalse(settings%io%write_residuals)
+    @assertTrue(settings%io%write_matrices)
   end subroutine test_parfile_dryrun
 
 

tests/unit_tests/test_parfiles/parfile_dryrun.par

@@ -5,4 +5,5 @@
 &savelist
   write_matrices = .true.
   write_eigenfunctions = .true.
+  write_derived_eigenfunctions = .true.
 /