From f0709454ae1ff0833b3816d8e567513146319191 Mon Sep 17 00:00:00 2001 From: yihau Date: Wed, 24 Jul 2024 22:04:17 +0800 Subject: [PATCH] unify benchmark result format --- .github/workflows/benchmark.yml | 55 +++++++++++++++++++++++++-------- ci/upload-benchmark.sh | 42 +++++++++++++++---------- 2 files changed, 67 insertions(+), 30 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a26f35e8b5f0ef..f0346589559aa2 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,36 +11,63 @@ jobs: strategy: fail-fast: false matrix: - # commnad should looks like cargo bench -- -Z unstable-options --format=json + # before adding your benchmark. please check these steps: + # 1. generate a file that includes your benchmark result. it should looks like + # + # ``` + # test bench_accounts_delta_hash ... bench: 48,035,858 ns/iter (+/- 2,118,806) + # ``` + # + # 2. run `DRY_RUN=1 ./ci/upload-benchmark.sh ` to ensure the datapoints are correct + # it should looks similar to this: + # + # ``` + # datapoint: ,commit=xxxx,test_suite=xxxx,name=bench_accounts_delta_hash median=48035858,deviation=2118806i + # ``` + # + # you only need to check `name`, `median` and `deviation` + # test: - { name: "solana-sdk", - command: "cargo +$rust_nightly bench -p solana-sdk -- -Z unstable-options --format=json", + commands: ["cargo +$rust_nightly bench -p solana-sdk"], } - { name: "solana-runtime", - command: "cargo +$rust_nightly bench -p solana-runtime -- -Z unstable-options --format=json", + commands: ["cargo +$rust_nightly bench -p solana-runtime"], } - { name: "solana-gossip", - command: "cargo +$rust_nightly bench -p solana-gossip -- -Z unstable-options --format=json", + commands: ["cargo +$rust_nightly bench -p solana-gossip"], } - { name: "solana-poh", - command: "cargo +$rust_nightly bench -p solana-poh -- -Z unstable-options --format=json", + commands: ["cargo +$rust_nightly bench -p solana-poh"], } - { name: "solana-core", - command: "cargo +$rust_nightly bench -p solana-core -- -Z unstable-options --format=json", - } - - { - name: "solana-accounts-db", - command: "cargo +$rust_nightly bench -p solana-accounts-db -- -Z unstable-options --format=json", + commands: ["cargo +$rust_nightly bench -p solana-core"], } - { name: "sbf", before_command: "make -C programs/sbf all", - command: "cargo +$rust_nightly bench --manifest-path programs/sbf/Cargo.toml --features=sbf_c -- -Z unstable-options --format=json", + commands: + [ + "cargo +$rust_nightly bench --manifest-path programs/sbf/Cargo.toml --features=sbf_c", + ], + } + # spliting solana-accounts-db because it includes criterion bench + - { + name: "solana-accounts-db", + commands: + [ + "cargo +$rust_nightly bench -p solana-accounts-db --bench accounts_index", + "cargo +$rust_nightly bench -p solana-accounts-db --bench accounts", + "cargo +$rust_nightly bench -p solana-accounts-db --bench append_vec", + "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_accounts_file -- --output-format bencher", + "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_hashing -- --output-format bencher", + "cargo +$rust_nightly bench -p solana-accounts-db --bench bench_serde -- --output-format bencher", + ], } steps: @@ -55,7 +82,9 @@ jobs: - name: Command run: | source ci/rust-version.sh nightly - ${{ matrix.test.command }} | tee benchmark.json + echo "${{ toJson(matrix.test.commands) }}" | jq -r '.[]' | while read command; do + eval $command | tee -a benchmark + done - name: Upload Result run: | @@ -66,4 +95,4 @@ jobs: INFLUX_USER="${{ secrets.BENCHMARK_INFLUX_USER }}" \ INFLUX_PASSWORD="${{ secrets.BENCHMARK_INFLUX_PASSWORD }}" \ INFLUX_MEASUREMENT="${{ secrets.BENCHMARK_INFLUX_MEASUREMENT }}" \ - ./ci/upload-benchmark.sh benchmark.json + ./ci/upload-benchmark.sh benchmark diff --git a/ci/upload-benchmark.sh b/ci/upload-benchmark.sh index ea475d24e0d309..c775cc38b1189b 100755 --- a/ci/upload-benchmark.sh +++ b/ci/upload-benchmark.sh @@ -17,6 +17,7 @@ REQUIRED ENVIRONMENTS: OPTIONAL ENVIRONMENTS: COMMIT_HASH Commit hash of the benchmark file TEST_SUITE The group name for all tests in the benchmark file + DRY_RUN Dry run ARGS: The output file generated by running @@ -53,28 +54,35 @@ if [ -z "$TEST_SUITE" ]; then TEST_SUITE="$(basename "${BENCHMARK_FILEPATH}")-$(date +%s)" fi -required_env_vars=( - "INFLUX_HOST" - "INFLUX_DB" - "INFLUX_USER" - "INFLUX_PASSWORD" - "INFLUX_MEASUREMENT" -) -for var in "${required_env_vars[@]}"; do - check_env "$var" -done +if [ -z "$DRY_RUN" ]; then + required_env_vars=( + "INFLUX_HOST" + "INFLUX_DB" + "INFLUX_USER" + "INFLUX_PASSWORD" + "INFLUX_MEASUREMENT" + ) + for var in "${required_env_vars[@]}"; do + check_env "$var" + done +fi while IFS= read -r line; do - type=$(echo "$line" | jq -r '.type') - if [ "$type" == "bench" ]; then - name=$(echo "$line" | jq -r '.name') - median=$(echo "$line" | jq -r '.median') - deviation=$(echo "$line" | jq -r '.deviation') + if [[ $line =~ ^test\ (.*)\ \.\.\.\ bench:\ *([0-9,]+)\ ns\/iter\ \(\+\/-\ *([0-9,]+)\) ]]; then + test_name="${BASH_REMATCH[1]}" + ns_iter="${BASH_REMATCH[2]}" + plus_minus="${BASH_REMATCH[3]}" + + ns_iter=$(echo "$ns_iter" | tr -d ',') + plus_minus=$(echo "$plus_minus" | tr -d ',') - datapoint="${INFLUX_MEASUREMENT},commit=${COMMIT_HASH},test_suite=${TEST_SUITE},name=${name} median=${median}i,deviation=${deviation}i" + datapoint="${INFLUX_MEASUREMENT},commit=${COMMIT_HASH},test_suite=${TEST_SUITE},name=${test_name} median=${ns_iter}i,deviation=${plus_minus}i" echo "datapoint: $datapoint" - curl -s -X POST "${INFLUX_HOST}/write?db=${INFLUX_DB}" --data-binary "$datapoint" + if [[ -z "$DRY_RUN" ]]; then + curl -s -X POST "${INFLUX_HOST}/write?db=${INFLUX_DB}" --data-binary "$datapoint" + fi fi + done <"$filepath"