unify benchmark result format

yihau · Jul 24, 2024 · f070945 · f070945
1 parent 0bca9e4
commit f070945
Show file tree

Hide file tree

Showing 2 changed files with 67 additions and 30 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -11,36 +11,63 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # commnad should looks like cargo bench -- -Z unstable-options --format=json
+        # before adding your benchmark. please check these steps:
+        # 1. generate a file that includes your benchmark result. it should looks like
+        #
+        #     ```
+        #     test bench_accounts_delta_hash                         ... bench:  48,035,858 ns/iter (+/- 2,118,806)
+        #     ```
+        #
+        # 2. run `DRY_RUN=1 ./ci/upload-benchmark.sh <YOUR_BENCHMARK_RESULT_FILE_PATH>` to ensure the datapoints are correct
+        #    it should looks similar to this:
+        #
+        #    ```
+        #    datapoint: ,commit=xxxx,test_suite=xxxx,name=bench_accounts_delta_hash median=48035858,deviation=2118806i
+        #    ```
+        #
+        #    you only need to check `name`, `median` and `deviation`
+        #
         test:
           - {
               name: "solana-sdk",
-              command: "cargo +$rust_nightly bench -p solana-sdk -- -Z unstable-options --format=json",
+              commands: ["cargo +$rust_nightly bench -p solana-sdk"],
             }
           - {
               name: "solana-runtime",
-              command: "cargo +$rust_nightly bench -p solana-runtime -- -Z unstable-options --format=json",
+              commands: ["cargo +$rust_nightly bench -p solana-runtime"],
             }
           - {
               name: "solana-gossip",
-              command: "cargo +$rust_nightly bench -p solana-gossip -- -Z unstable-options --format=json",
+              commands: ["cargo +$rust_nightly bench -p solana-gossip"],
             }
           - {
               name: "solana-poh",
-              command: "cargo +$rust_nightly bench -p solana-poh -- -Z unstable-options --format=json",
+              commands: ["cargo +$rust_nightly bench -p solana-poh"],
             }
           - {
               name: "solana-core",
-              command: "cargo +$rust_nightly bench -p solana-core -- -Z unstable-options --format=json",
-            }
-          - {
-              name: "solana-accounts-db",
-              command: "cargo +$rust_nightly bench -p solana-accounts-db -- -Z unstable-options --format=json",
+              commands: ["cargo +$rust_nightly bench -p solana-core"],
             }
           - {
               name: "sbf",
               before_command: "make -C programs/sbf all",
-              command: "cargo +$rust_nightly bench --manifest-path programs/sbf/Cargo.toml --features=sbf_c -- -Z unstable-options --format=json",
+              commands:
+                [
+                  "cargo +$rust_nightly bench --manifest-path programs/sbf/Cargo.toml --features=sbf_c",
+                ],
+            }
+          # spliting solana-accounts-db because it includes criterion bench
+          - {
+              name: "solana-accounts-db",
+              commands:
+                [
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench accounts_index",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench accounts",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench append_vec",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_accounts_file -- --output-format bencher",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_hashing -- --output-format bencher",
+                  "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_serde -- --output-format bencher",
+                ],
             }
 
     steps:
@@ -55,7 +82,9 @@ jobs:
       - name: Command
         run: |
           source ci/rust-version.sh nightly
-          ${{ matrix.test.command }} | tee benchmark.json
+          echo "${{ toJson(matrix.test.commands) }}" | jq -r '.[]' | while read command; do
+            eval $command | tee -a benchmark
+          done
 
       - name: Upload Result
         run: |
@@ -66,4 +95,4 @@ jobs:
           INFLUX_USER="${{ secrets.BENCHMARK_INFLUX_USER }}" \
           INFLUX_PASSWORD="${{ secrets.BENCHMARK_INFLUX_PASSWORD }}" \
           INFLUX_MEASUREMENT="${{ secrets.BENCHMARK_INFLUX_MEASUREMENT }}" \
-          ./ci/upload-benchmark.sh benchmark.json
+          ./ci/upload-benchmark.sh benchmark
diff --git a/ci/upload-benchmark.sh b/ci/upload-benchmark.sh
@@ -17,6 +17,7 @@ REQUIRED ENVIRONMENTS:
 OPTIONAL ENVIRONMENTS:
     COMMIT_HASH   Commit hash of the benchmark file
     TEST_SUITE    The group name for all tests in the benchmark file
+    DRY_RUN       Dry run
 
 ARGS:
     <BENCHMARK_FILEPATH>    The output file generated by running
@@ -53,28 +54,35 @@ if [ -z "$TEST_SUITE" ]; then
   TEST_SUITE="$(basename "${BENCHMARK_FILEPATH}")-$(date +%s)"
 fi
 
-required_env_vars=(
-  "INFLUX_HOST"
-  "INFLUX_DB"
-  "INFLUX_USER"
-  "INFLUX_PASSWORD"
-  "INFLUX_MEASUREMENT"
-)
-for var in "${required_env_vars[@]}"; do
-  check_env "$var"
-done
+if [ -z "$DRY_RUN" ]; then
+  required_env_vars=(
+    "INFLUX_HOST"
+    "INFLUX_DB"
+    "INFLUX_USER"
+    "INFLUX_PASSWORD"
+    "INFLUX_MEASUREMENT"
+  )
+  for var in "${required_env_vars[@]}"; do
+    check_env "$var"
+  done
+fi
 
 while IFS= read -r line; do
-  type=$(echo "$line" | jq -r '.type')
 
-  if [ "$type" == "bench" ]; then
-    name=$(echo "$line" | jq -r '.name')
-    median=$(echo "$line" | jq -r '.median')
-    deviation=$(echo "$line" | jq -r '.deviation')
+  if [[ $line =~ ^test\ (.*)\ \.\.\.\ bench:\ *([0-9,]+)\ ns\/iter\ \(\+\/-\ *([0-9,]+)\) ]]; then
+    test_name="${BASH_REMATCH[1]}"
+    ns_iter="${BASH_REMATCH[2]}"
+    plus_minus="${BASH_REMATCH[3]}"
+
+    ns_iter=$(echo "$ns_iter" | tr -d ',')
+    plus_minus=$(echo "$plus_minus" | tr -d ',')
 
-    datapoint="${INFLUX_MEASUREMENT},commit=${COMMIT_HASH},test_suite=${TEST_SUITE},name=${name} median=${median}i,deviation=${deviation}i"
+    datapoint="${INFLUX_MEASUREMENT},commit=${COMMIT_HASH},test_suite=${TEST_SUITE},name=${test_name} median=${ns_iter}i,deviation=${plus_minus}i"
     echo "datapoint: $datapoint"
 
-    curl -s -X POST "${INFLUX_HOST}/write?db=${INFLUX_DB}" --data-binary "$datapoint"
+    if [[ -z "$DRY_RUN" ]]; then
+      curl -s -X POST "${INFLUX_HOST}/write?db=${INFLUX_DB}" --data-binary "$datapoint"
+    fi
   fi
+
 done <"$filepath"