Display test results as a badge and action job summary

A single build executes ~9 million tests in order to cover the
api, feature matrix, linearization, Guava compatibility, and 98^
code coverage. A build is performed on JDK 11 and 18 (x86) and
17 (arm64), where the ARM run assert that there are no latent
bugs due to a weaker memory model. This takes ~2.5 hours despite
using parallel tests and jobs.

As it is hard to track our test size, the JDK 18 run is used as
the reference. The test results are aggregated and published to
have some visibility. We do not count the JDK 11 x86 and JDK 17
arm64 runs as repeats.

Author: ben-manes

.github/scripts/test.sh

@@ -8,6 +8,14 @@ GUAVA_STRONG_KEYS=
 GUAVA_WEAK_KEYS=
 GUAVA_CHECK=
 
+if [[ -z "$@" ]]; then
+  echo "Please specify the test suites to run"
+  echo "  --caffeine=[strongKeys, weakKeys, check]"
+  echo "  --guava=[strongKeys, weakKeys, check]"
+  echo "  --all"
+  exit 1
+fi
+
 for i in "$@"; do
   case $i in
     --caffeine=strongKeys)
@@ -37,7 +45,11 @@ for i in "$@"; do
       GUAVA_CHECK=true
       ;;
     -*)
-      echo "Unknown option $i; --caffeine=? or --guava=? with strongKeys, weakKeys, or check"
+      echo "Please specify the test suites to run"
+      echo "  --caffeine=[strongKeys, weakKeys, check]"
+      echo "  --guava=[strongKeys, weakKeys, check]"
+      echo "  --all"
+      echo "Unknown option $i"
       exit 1
       ;;
   esac

.github/workflows/build.yml

@@ -57,21 +57,40 @@ jobs:
       - name: Setup Gradle
         uses: gradle/gradle-build-action@v2
       - name: Run tests
-        run: ./.github/scripts/test.sh --caffeine=strongKeys --caffeine=weakKeys --caffeine=check
+        run: |
+          stopMarker=$(uuidgen)
+          echo "::stop-commands::$stopMarker"
+          trap "echo '::$stopMarker::'" EXIT
+          ./.github/scripts/test.sh \
+            --caffeine=strongKeys \
+            --caffeine=weakKeys \
+            --caffeine=check
+      - name: Compress test results
+        uses: sibiraj-s/action-archiver@v1
+        if: always() && (matrix.java == env.MAX_JVM)
+        with:
+          path: '**/results/*.xml'
+          output: caffeine.tar.gz
+          format: tar
+          gzip: true
       - name: Upload test results
         uses: actions/upload-artifact@v3
-        if: (matrix.java == env.MAX_JVM) && always()
+        if: always() && (matrix.java == env.MAX_JVM)
         with:
           retention-days: 1
           name: caffeine-test-results
-          path: '**/results/*.xml'
+          path: caffeine.tar.gz
       - name: Publish Coverage
         if: >
           matrix.java == env.MAX_JVM
           && github.event_name == 'push'
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
-        run: ./gradlew --daemon coveralls -S
+        run: |
+          stopMarker=$(uuidgen)
+          echo "::stop-commands::$stopMarker"
+          trap "echo '::$stopMarker::'" EXIT
+          ./gradlew --daemon coveralls -S
         continue-on-error: true
       - name: SonarQube
         if: >
@@ -81,6 +100,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
+          stopMarker=$(uuidgen)
+          echo "::stop-commands::$stopMarker"
+          trap "echo '::$stopMarker::'" EXIT
           git fetch --unshallow
           ./gradlew --daemon sonarqube -S -Dsonar.branch.name=${GITHUB_REF##*/}
         continue-on-error: true
@@ -129,14 +151,29 @@ jobs:
       - name: Setup Gradle
         uses: gradle/gradle-build-action@v2
       - name: Run tests
-        run: ./.github/scripts/test.sh --guava=strongKeys --guava=weakKeys --guava=check
+        run: |
+          stopMarker=$(uuidgen)
+          echo "::stop-commands::$stopMarker"
+          trap "echo '::$stopMarker::'" EXIT
+          ./.github/scripts/test.sh \
+            --guava=strongKeys \
+            --guava=weakKeys \
+            --guava=check
+      - name: Compress test results
+        uses: sibiraj-s/action-archiver@v1
+        if: always() && (matrix.java == env.MAX_JVM)
+        with:
+          path: '**/results/*.xml'
+          output: guava.tar.gz
+          format: tar
+          gzip: true
       - name: Upload test results
         uses: actions/upload-artifact@v3
-        if: (matrix.java == env.MAX_JVM) && always()
+        if: always() && (matrix.java == env.MAX_JVM)
         with:
           retention-days: 1
           name: guava-test-results
-          path: '**/results/*.xml'
+          path: guava.tar.gz
       - name: Stop Gradle daemon
         if: always()
         run: ./gradlew --stop
@@ -145,12 +182,91 @@ jobs:
         continue-on-error: true
         if: failure()
 
-  event_file:
-    name: Event File
+  lincheck:
+    name: Linearization Tests
     runs-on: ubuntu-latest
     steps:
-    - name: Upload
-      uses: actions/upload-artifact@v2
-      with:
-        name: Event File
-        path: ${{ github.event_path }}
+      - uses: actions/checkout@v3
+      - name: Set up JDK ${{ env.MAX_JVM }}
+        uses: actions/setup-java@v3
+        with:
+          cache: 'gradle'
+          distribution: 'zulu'
+          java-version: ${{ env.MAX_JVM }}
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+      - name: Run tests
+        run: |
+          stopMarker=$(uuidgen)
+          echo "::stop-commands::$stopMarker"
+          trap "echo '::$stopMarker::'" EXIT
+          JAVA_VERSION=$MAX_JVM
+          ./gradlew lincheckTest
+      - name: Compress test results
+        uses: sibiraj-s/action-archiver@v1
+        if: always()
+        with:
+          path: '**/results/*.xml'
+          output: lincheck.tar.gz
+          format: tar
+          gzip: true
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          retention-days: 1
+          name: lincheck-test-results
+          path: lincheck.tar.gz
+
+  test-summary:
+    name: Test Summary
+    runs-on: ubuntu-latest
+    needs: [caffeine, guava, lincheck]
+    permissions:
+      checks: write
+    steps:
+      - name: Download Guava Tests
+        uses: actions/download-artifact@v3
+        with:
+          name: guava-test-results
+      - name: Download Caffeine Tests
+        uses: actions/download-artifact@v3
+        with:
+          name: caffeine-test-results
+      - name: Download Lincheck Tests
+        uses: actions/download-artifact@v3
+        with:
+          name: lincheck-test-results
+      - name: Decompress
+        run: |
+          mkdir guava && tar -zxf guava.tar.gz --directory guava
+          mkdir caffeine && tar -zxf caffeine.tar.gz --directory caffeine
+          mkdir lincheck && tar -zxf lincheck.tar.gz --directory lincheck
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        continue-on-error: true
+        id: test-results
+        with:
+          comment_mode: off
+          ignore_runs: true
+          job_summary: true
+          files: '**/*.xml'
+      - name: Format Test Count
+        run: |
+          LANG=en_US.utf8
+          COUNT=${{ fromJSON(steps.test-results.outputs.json).stats.runs }}
+          echo "TEST_COUNT=$(printf "%'d" $COUNT)" >> $GITHUB_ENV
+      - name: Create badge
+        uses: emibcn/badge-action@d6f51ff11b5c3382b3b88689ae2d6db22d9737d1
+        with:
+          label: tests
+          color: 31c653
+          path: badge.svg
+          status: ${{ env.TEST_COUNT }}
+      - name: Upload badge to Gist
+        uses: andymckay/append-gist-action@1fbfbbce708a39bd45846f0955ed5521f2099c6d
+        if: endsWith(github.ref, github.event.repository.default_branch)
+        with:
+          gistURL: https://gist.githubusercontent.com/ben-manes/c20eb418f0e0bd6dfe1c25beb35faae4
+          token: ${{ secrets.GIST_TOKEN }}
+          file: badge.svg

.github/workflows/lincheck.yml

@@ -1,9 +1,10 @@
 [![Build Status](https://github.com/ben-manes/caffeine/workflows/build/badge.svg)](https://github.com/ben-manes/caffeine/actions?query=workflow%3Abuild)
+[![Test Count](https://gist.githubusercontent.com/ben-manes/c20eb418f0e0bd6dfe1c25beb35faae4/raw/badge.svg)](https://github.com/ben-manes/caffeine/actions?query=workflow%3Abuild)
 [![Coverage Status](https://img.shields.io/coveralls/ben-manes/caffeine.svg)](https://coveralls.io/r/ben-manes/caffeine?branch=master)
 [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.github.ben-manes.caffeine/caffeine/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.github.ben-manes.caffeine/caffeine)
-[![JavaDoc](http://www.javadoc.io/badge/com.github.ben-manes.caffeine/caffeine.svg)](http://www.javadoc.io/doc/com.github.ben-manes.caffeine/caffeine)
-[![License](http://img.shields.io/:license-apache-brightgreen.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)
-[![Stack Overflow](http://img.shields.io/:stack%20overflow-caffeine-brightgreen.svg)](http://stackoverflow.com/questions/tagged/caffeine)
+[![JavaDoc](https://www.javadoc.io/badge/com.github.ben-manes.caffeine/caffeine.svg)](http://www.javadoc.io/doc/com.github.ben-manes.caffeine/caffeine)
+[![License](https://img.shields.io/:license-apache-brightgreen.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)
+[![Stack Overflow](https://img.shields.io/:stack%20overflow-caffeine-brightgreen.svg)](http://stackoverflow.com/questions/tagged/caffeine)
 <a href="https://github.com/ben-manes/caffeine/wiki">
 <img align="right" height="90px" src="https://raw.githubusercontent.com/ben-manes/caffeine/master/wiki/logo.png">
 </a>

.github/workflows/test-report.yml

@@ -77,7 +77,7 @@ ext {
     mockito: '4.5.1',
     paxExam: '4.13.5',
     slf4jTest: '2.6.1',
-    testng: '7.5',
+    testng: '7.6.0',
     truth: '1.1.3',
     felix: '7.0.3',
     felixScr: '2.2.0',