diff --git a/.travis.yml b/.travis.yml index 923835bc..ff42198e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,17 +9,17 @@ go: env: global: - GOTFLAGS="-race" - matrix: - BUILD_DEPTYPE=gomod - + matrix: + - TEST_PHASE=test + - TEST_PHASE=benchmark # disable travis install install: - true script: - - bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh) - + - ./bin/test-exec.sh cache: directories: diff --git a/bin/benchmark-to-file.sh b/bin/benchmark-to-file.sh new file mode 100755 index 00000000..d1aabbaa --- /dev/null +++ b/bin/benchmark-to-file.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +git checkout "$2" + +IPFS_LOGGING=critical go test -benchtime=3s -run=NONE -bench=. ./... | tee "$1" diff --git a/bin/diff-benchmarks.sh b/bin/diff-benchmarks.sh new file mode 100755 index 00000000..8907d747 --- /dev/null +++ b/bin/diff-benchmarks.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +parse() { + sed -n \ + -e 's/ *\t */\t/g' \ + -e '/^Benchmark/p' | + awk 'BEGIN{print "{\"results\": ["} {print " {\"name\": \"",$1,"\", \"time\": ",$3," },"} END{print "]}"}' OFS="" ORS=" "| + sed -e 's/, ]/ ]/g' | + jq '.results[] | {name: .name, time: .time }' + +} + +benchcmp "$1" "$2" + +echo "" +echo "Result:" + +{ + parse < "$1" + parse < "$2" +} | jq -e -r -s 'group_by(.name)[] | {name: .[0].name, speedup: (.[0].time / .[1].time)} | select(.speedup < 0.75) | "\(.name)\t\(.speedup)x"' + +if [[ $? -ne 4 ]]; then + echo "" + echo "FAIL" + exit 1 +else + echo "PASS" +fi diff --git a/bin/run-benchmarks.sh b/bin/run-benchmarks.sh new file mode 100755 index 00000000..0e477b8a --- /dev/null +++ b/bin/run-benchmarks.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -eo pipefail + +display_and_run() { + echo "***" "$@" + eval "$(printf '%q ' "$@")" +} + +# reset workdir to state from git (to remove possible rewritten dependencies) +export GO111MODULE=on +display_and_run go get golang.org/x/tools/cmd/benchcmp +display_and_run git reset --hard +git checkout -b after +git fetch origin master:refs/remotes/origin/before +git checkout remotes/origin/before +git checkout -b before +git checkout after +display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-before.txt before +git checkout after +display_and_run BENCHMARK_SEED="$$" ./bin/benchmark-to-file.sh benchmark-after.txt after +if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then +display_and_run ./bin/diff-benchmarks.sh benchmark-before.txt benchmark-after.txt +fi diff --git a/bin/test-exec.sh b/bin/test-exec.sh new file mode 100755 index 00000000..506aa66b --- /dev/null +++ b/bin/test-exec.sh @@ -0,0 +1,6 @@ +#!/bin/bash +if [[ "$TEST_PHASE" == "test" ]]; then + bash <(curl -s https://raw.githubusercontent.com/ipfs/ci-helpers/master/travis-ci/run-standard-tests.sh) +else + ./bin/run-benchmarks.sh +fi \ No newline at end of file diff --git a/go.mod b/go.mod index 1bafbdc6..feb07e44 100644 --- a/go.mod +++ b/go.mod @@ -36,5 +36,6 @@ require ( golang.org/x/net v0.0.0-20190611141213-3f473d35a33a // indirect golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae // indirect golang.org/x/text v0.3.2 // indirect + golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac // indirect gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index c80ee5bb..248e0c4b 100644 --- a/go.sum +++ b/go.sum @@ -300,11 +300,13 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7 h1:C2F/nMkR/9sfUTpvR3QrjBuTdvMUC/cFajkphs1YLQo= golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a h1:+KkCgOMgnKSgenxTBoiwkMqTiouMIy/3o8RLdmSbGoY= golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -320,6 +322,10 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d h1:LQ06Vbju+Kwbcd94hb+6CgDsWoj/e7GOLPcYzHrG+iI= +golang.org/x/tools v0.0.0-20190619215442-4adf7a708c2d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=