benchmark for SubscribeAsync

.github/workflows/build-test.yml

@@ -48,7 +48,11 @@ on:
         default: "OFF"
       verbose_make_output:
         type: string
-        default: "ON"
+        default: "OFF"
+      benchmark:
+        type: string
+        default: "OFF"
+
     secrets:
       CODECOV_TOKEN:
         description: "Codecov repo token"
@@ -129,15 +133,6 @@ jobs:
           cmake .. ${{ steps.cmake-flags.outputs.flags }}
           make rebuild_cache && make
 
-      - name: "Rebuild the list of tests to match the compile flags"
-        working-directory: ./build
-        run: |
-          ./bin/testsuite
-          if [[ $(diff list.txt ../test/list.txt; echo $?) != 0 ]]; then
-            mv list.txt ../test/list.txt
-            make rebuild_cache
-          fi
-
       # testing
 
       - name: "Download nats-server version ${{ inputs.server_version }}"
@@ -168,12 +163,13 @@ jobs:
           fi
 
       - name: "Test"
+        if: inputs.benchmark == 'OFF'
         working-directory: ./build
         run: |
           export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH
           export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
           flags=""
-          ctest --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }}
+          ctest -L 'test' --timeout 60 --output-on-failure --repeat-until-fail ${{ inputs.repeat }}
 
       - name: Upload coverage reports to Codecov
         # PRs from external contributors fail: https://github.com/codecov/feedback/issues/301
@@ -183,4 +179,42 @@ jobs:
         with:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
-          # verbose: true
+          # verbose: true
+
+      - name: "Download benchmark results from ${{ github.event.pull_request.base.ref }}"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.base.ref
+        uses: actions/download-artifact@v2
+        with:
+          name: benchmark_results_${{ github.event.pull_request.base.ref }}
+          path: ./build/prev_bench.log
+        continue-on-error: true
+
+      - name: "Benchmark"
+        if: inputs.benchmark == 'ON'
+        working-directory: ./build
+        run: |
+          export PATH=../deps/nats-server:../deps/nats-streaming-server:$PATH
+          export NATS_TEST_SERVER_VERSION="$(nats-server -v)"
+          flags=""
+          ctest -L 'bench' --timeout 600 -VV | tee bench.log
+          #
+          # ...coming: compare to base branch
+
+      - name: "Upload benchmark result for PR ${{ github.event.pull_request.head.ref }}"
+        if: inputs.benchmark == 'ON' && github.event.pull_request.head.ref
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_results_${{ github.event.pull_request.head.ref }}
+          path: ./build/bench.log
+
+      - name: Extract branch name
+        if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref
+        id: extract_branch
+        run: echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+
+      - name: "Upload benchmark result for branch ${{ env.BRANCH_NAME }}"
+        if: inputs.benchmark == 'ON' && !github.event.pull_request.head.ref
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark_results_${{ env.BRANCH_NAME }}
+          path: ./build/bench.log

.github/workflows/on-pr-debug.yml

@@ -1,4 +1,4 @@
-name: "Debug"
+name: "PR"
 on:
   pull_request:
 
@@ -29,7 +29,7 @@ jobs:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   coverage-pooled:
-    name: "Coverage"
+    name: "Coverage (pooled delivery)"
     uses: ./.github/workflows/build-test.yml
     with:
       coverage: ON
@@ -74,6 +74,14 @@ jobs:
       server_version: main
       lib_write_deadline: ON
 
+  bench:
+    name: "Benchmark"
+    uses: ./.github/workflows/build-test.yml
+    with:
+      server_version: main
+      benchmark: ON
+      type: Release
+
   Windows:
     name: "Windows"
     runs-on: windows-latest
@@ -98,4 +106,4 @@ jobs:
       - name: Test
         run: |
           cd build
-          ./bin/Debug/testsuite
+          ctest -L 'test' --timeout 60 --output-on-failure

.github/workflows/on-push-release-extra.yml

@@ -5,7 +5,6 @@ on:
       - main
       - release_*
 
-
 permissions:
   contents: write # required by build-test to comment on coverage but not used here.
 
@@ -67,3 +66,11 @@ jobs:
       type: Debug
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  bench:
+    name: "Benchmark"
+    uses: ./.github/workflows/build-test.yml
+    with:
+      server_version: main
+      benchmark: ON
+      type: Release

README.md

@@ -241,38 +241,15 @@ ctest -T memcheck -V -I 1,4
 ```
 The above command would run the tests with `valgrind` (`-T memcheck`), with verbose output (`-V`), and run the tests from 1 to 4 (`-I 1,4`).
 
-If you add a test to `test/test.c`, you need to add it into the `allTests` array. Each entry contains a name, and the test function. You can add it anywhere into this array.
-Build you changes:
+If you add a test to `test/test.c`, you need to add it into the `list_test.txt`
+file. Each entry contains just the test name, the function must be named
+identically, with a `test_` prefix. The list is in alphabetical order, but it
+does not need to be, you can add anywhere.
 
-```
-make
-[ 44%] Built target nats
-[ 88%] Built target nats_static
-[ 90%] Built target nats-publisher
-[ 92%] Built target nats-queuegroup
-[ 94%] Built target nats-replier
-[ 96%] Built target nats-requestor
-[ 98%] Built target nats-subscriber
-Scanning dependencies of target testsuite
-[100%] Building C object test/CMakeFiles/testsuite.dir/test.c.o
-Linking C executable testsuite
-[100%] Built target testsuite
-```
-
-Now regenerate the list by invoking the test suite without any argument:
-
-```
-./test/testsuite
-Number of tests: 77
-```
-
-This list the number of tests added to the file `list.txt`. Move this file to the source's test directory.
-
-```
-mv list.txt ../test/
-```
+If you are adding a benchmark, it should be added to the `list_bench.txt`. These
+tests are labeled differently (`-L 'bench'`) and executed separately on CI.
 
-Then, refresh the build:
+You need to re-run `cmake` for the changes to take effect:
 
 ```
 cmake ..

buildOnTravis.sh

@@ -75,7 +75,7 @@ fi
 
 export NATS_TEST_TRAVIS=yes
 echo "Using NATS server version: $NATS_TEST_SERVER_VERSION"
-ctest --timeout 60 --output-on-failure $4
+ctest -L 'test' --timeout 60 --output-on-failure $4
 res=$?
 if [ $res -ne 0 ]; then
   exit $res

test/CMakeLists.txt

@@ -1,9 +1,12 @@
+cmake_minimum_required(VERSION 3.10)
+
 if(NOT BUILD_TESTING)
   return()
 endif()
+
 if(NOT NATS_BUILD_LIB_STATIC)
-   MESSAGE(FATAL_ERROR
-     "Building tests require static library, or run CMake with -DBUILD_TESTING=OFF")
+  MESSAGE(FATAL_ERROR
+    "Building tests require static library, or run CMake with -DBUILD_TESTING=OFF")
   return()
 endif()
 
@@ -13,45 +16,62 @@ endif()
 
 # We need this to build the test program
 include_directories(${PROJECT_SOURCE_DIR}/src)
+
 if(NATS_BUILD_WITH_TLS)
   include_directories(${OPENSSL_INCLUDE_DIR})
 endif(NATS_BUILD_WITH_TLS)
+
 if(NATS_BUILD_STREAMING)
   include_directories(${NATS_PROTOBUF_INCLUDE_DIRS})
   include_directories(${PROJECT_SOURCE_DIR}/src/stan)
 endif(NATS_BUILD_STREAMING)
 
 # Build the test program
-add_executable(testsuite test.c)
+add_executable(testsuite test.c bench_sub_async.c)
 
 # Link statically with the library
 target_link_libraries(testsuite nats_static ${NATS_EXTRA_LIB})
 
-# Set the test index to 0
-set(testIndex 0)
+set(BENCH_LIST ${PROJECT_SOURCE_DIR}/test/list_bench.txt)
+set(STAN_LIST ${PROJECT_SOURCE_DIR}/test/list_stan.txt)
 
-# Read the file 'list.txt' to get all the test names
-file(STRINGS list.txt listOfTestNames)
+list(APPEND ALL_LISTS ${PROJECT_SOURCE_DIR}/test/list_test.txt)
+list(APPEND ALL_LISTS ${BENCH_LIST})
+if(NATS_BUILD_STREAMING)
+list(APPEND ALL_LISTS ${STAN_LIST})
+message("Building Streaming tests" ${ALL_LISTS})
+endif()
 
-# For each test name
-foreach(name ${listOfTestNames})
-  # Create a test and pass the index (start and end are the same)
-  # to the testsuite executable
-  add_test(NAME Test_${name}
-           WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-           COMMAND testsuite ${testIndex} ${testIndex})
+set(TEST_NAMES)
 
-  # Make sure the test passes
-  set_tests_properties(Test_${name} PROPERTIES PASS_REGULAR_EXPRESSION "ALL PASSED")
+foreach(LIST_FILE ${ALL_LISTS})
+  # Get all the test names
+  if(EXISTS ${LIST_FILE})
+    file(STRINGS ${LIST_FILE} TEST_NAMES)
+  else()
+    set(TEST_NAMES)
+  endif()
 
-  # Set TSAN_OPTIONS for the test
-  if(NATS_SANITIZE)
-    set_tests_properties(Test_${name} PROPERTIES
-           ENVIRONMENT "TSAN_OPTIONS=detect_deadlocks=1:second_deadlock_stack=1:halt_on_error=1:report_signal_unsafe=1")
-  endif(NATS_SANITIZE)
+  foreach(name ${TEST_NAMES})
+    # Remove the _test() prefix
+    string(REGEX REPLACE "_test\\(([^)]+)\\)" "\\1" TEST_NAME ${name})
+    add_test(NAME ${TEST_NAME}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+      COMMAND testsuite ${TEST_NAME})
 
-  # Bump the test index number
-  math(EXPR testIndex "${testIndex}+1")
-endforeach()
+    if(${LIST_FILE} STREQUAL ${BENCH_LIST})
+      set_tests_properties(${TEST_NAME} PROPERTIES LABELS "bench")
+    else()
+      # Make sure the test passes
+      set_tests_properties(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION "ALL PASSED")
 
+      # Set TSAN_OPTIONS for the test
+      if(NATS_SANITIZE)
+        set_tests_properties(${TEST_NAME} PROPERTIES
+          ENVIRONMENT "TSAN_OPTIONS=detect_deadlocks=1:second_deadlock_stack=1:halt_on_error=1:report_signal_unsafe=1")
+      endif(NATS_SANITIZE)
 
+      set_tests_properties(${TEST_NAME} PROPERTIES LABELS "test")
+    endif()
+  endforeach()
+endforeach()