moved bench to test

refactored bench
Added BenchSubscribeAsync_Inject
Added BenchSubscribeAsync_InjectSlow

.github/workflows/build-test.yml

@@ -133,15 +133,6 @@ jobs:
           cmake .. ${{ steps.cmake-flags.outputs.flags }}
           make rebuild_cache && make
 
-      - name: "Rebuild the list of tests to match the compile flags"
-        working-directory: ./build
-        run: |
-          ./bin/testsuite
-          if [[ $(diff list.txt ../test/list.txt; echo $?) != 0 ]]; then
-            mv list.txt ../test/list.txt
-            make rebuild_cache
-          fi
-
       # testing
 
       - name: "Download nats-server version ${{ inputs.server_version }}"
@@ -178,7 +169,7 @@ jobs:
           export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH
           export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
           flags=""
-          ctest --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }}
+          ctest -L 'test' --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }}
 
       - name: Upload coverage reports to Codecov
         # PRs from external contributors fail: https://github.com/codecov/feedback/issues/301
@@ -190,10 +181,40 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           # verbose: true
 
+      - name: "Download benchmark results from ${{ github.event.pull_request.base.ref }}"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref
+        uses: actions/download-artifact@v2
+        with:
+          name: benchmark_results_${{ github.event.pull_request.base.ref }}
+          path: ./build/prev_bench.log
+        continue-on-error: true
+
       - name: "Benchmark"
         if: inputs.benchmark == 'ON'
         working-directory: ./build
         run: |
           export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH
           export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
-          ./bin/bench-sub-async
+          flags=""
+          ctest -L 'bench' --timeout 600 -VV | tee bench.log
+          #
+          # ...coming: compare to base branch
+
+      - name: "Upload benchmark result for PR ${{ github.event.pull_request.head.ref }}"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.head.ref
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_results_${{ github.event.pull_request.head.ref }}
+          path: ./build/bench.log
+
+      - name: Extract branch name
+        if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref
+        id: extract_branch
+        run: echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+
+      - name: "Upload benchmark result for branch ${{ env.BRANCH_NAME }}"
+        if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_results_${{ env.BRANCH_NAME }}
+          path: ./build/bench.log

.github/workflows/on-pr-debug.yml

@@ -1,4 +1,4 @@
-name: "Debug"
+name: "PR"
 on:
   pull_request:
 
@@ -29,7 +29,7 @@ jobs:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   coverage-pooled:
-    name: "Coverage"
+    name: "Coverage (pooled delivery)"
     uses: ./.github/workflows/build-test.yml
     with:
       coverage: ON
@@ -74,6 +74,14 @@ jobs:
       server_version: main
       lib_write_deadline: ON
 
+  bench:
+    name: "Benchmark"
+    uses: ./.github/workflows/build-test.yml
+    with:
+      server_version: main
+      benchmark: ON
+      type: Release
+
   Windows:
     name: "Windows"
     runs-on: windows-latest
@@ -98,4 +106,4 @@ jobs:
       - name: Test
         run: |
           cd build
-          ./bin/Debug/testsuite
+          ctest -L 'test' --timeout 60 --output-on-failure

.github/workflows/on-push-release-extra.yml

@@ -5,7 +5,6 @@ on:
       - main
       - release_*
 
-
 permissions:
   contents: write # required by build-test to comment on coverage but not used here.
 
@@ -67,3 +66,11 @@ jobs:
       type: Debug
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  bench:
+    name: "Benchmark"
+    uses: ./.github/workflows/build-test.yml
+    with:
+      server_version: main
+      benchmark: ON
+      type: Release

.github/workflows/on-push-release.yml

@@ -45,11 +45,3 @@ jobs:
     with:
       sanitize: thread
       server_version: main
-
-  bench:
-    name: "Benchmark"
-    uses: ./.github/workflows/build-test.yml
-    with:
-      server_version: main
-      benchmark: ON
-      type: Release

CMakeLists.txt

@@ -283,7 +283,6 @@ add_subdirectory(examples/getstarted)
 if(NATS_BUILD_STREAMING)
   add_subdirectory(examples/stan)
 endif()
-add_subdirectory(bench)
 add_subdirectory(test)
 add_subdirectory(test/dylib)
 #----------------------------

README.md

@@ -241,38 +241,15 @@ ctest -T memcheck -V -I 1,4
 ```
 The above command would run the tests with `valgrind` (`-T memcheck`), with verbose output (`-V`), and run the tests from 1 to 4 (`-I 1,4`).
 
-If you add a test to `test/test.c`, you need to add it into the `allTests` array. Each entry contains a name, and the test function. You can add it anywhere into this array.
-Build you changes:
+If you add a test to `test/test.c`, you need to add it into the `list_test.txt`
+file. Each entry contains just the test name, the function must be named
+identically, with a `test_` prefix. The list is in alphabetical order, but it
+does not need to be, you can add anywhere.
 
-```
-make
-[ 44%] Built target nats
-[ 88%] Built target nats_static
-[ 90%] Built target nats-publisher
-[ 92%] Built target nats-queuegroup
-[ 94%] Built target nats-replier
-[ 96%] Built target nats-requestor
-[ 98%] Built target nats-subscriber
-Scanning dependencies of target testsuite
-[100%] Building C object test/CMakeFiles/testsuite.dir/test.c.o
-Linking C executable testsuite
-[100%] Built target testsuite
-```
-
-Now regenerate the list by invoking the test suite without any argument:
-
-```
-./test/testsuite
-Number of tests: 77
-```
-
-This list the number of tests added to the file `list.txt`. Move this file to the source's test directory.
-
-```
-mv list.txt ../test/
-```
+If you are adding a benchmark, it should be added to the `list_bench.txt`. These
+tests are labeled differently (`-L 'bench'`) and executed separately on CI.
 
-Then, refresh the build:
+You need to re-run `cmake` for the changes to take effect:
 
 ```
 cmake ..