Benchmarking #1497
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarking | |
| on: | |
| # uncomment to run on push for debugging your PR | |
| # push: | |
| # branches: [ your branch ] | |
| schedule: | |
| # * is a special character in YAML so you have to quote this string | |
| # ┌───────────── minute (0 - 59) | |
| # │ ┌───────────── hour (0 - 23) | |
| # │ │ ┌───────────── day of the month (1 - 31) | |
| # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) | |
| # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) | |
| - cron: '0 3 * * *' # run once a day, timezone is utc | |
| workflow_dispatch: # adds ability to run this manually | |
| inputs: | |
| region_id: | |
| description: 'Project region id. If not set, the default region will be used' | |
| required: false | |
| default: 'aws-us-east-2' | |
| save_perf_report: | |
| type: boolean | |
| description: 'Publish perf report. If not set, the report will be published only for the main branch' | |
| required: false | |
| collect_olap_explain: | |
| type: boolean | |
| description: 'Collect EXPLAIN ANALYZE for OLAP queries. If not set, EXPLAIN ANALYZE will not be collected' | |
| required: false | |
| default: false | |
| collect_pg_stat_statements: | |
| type: boolean | |
| description: 'Collect pg_stat_statements for OLAP queries. If not set, pg_stat_statements will not be collected' | |
| required: false | |
| default: false | |
| run_AWS_RDS_AND_AURORA: | |
| type: boolean | |
| description: 'AWS-RDS and AWS-AURORA normally only run on Saturday. Set this to true to run them on every workflow_dispatch' | |
| required: false | |
| default: false | |
| run_only_pgvector_tests: | |
| type: boolean | |
| description: 'Run pgvector tests but no other tests. If not set, all tests including pgvector tests will be run' | |
| required: false | |
| default: false | |
| defaults: | |
| run: | |
| shell: bash -euxo pipefail {0} | |
| concurrency: | |
| # Allow only one workflow per any non-`main` branch. | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} | |
| cancel-in-progress: true | |
| jobs: | |
| pgbench-pgvector: | |
| env: | |
| TEST_PG_BENCH_DURATIONS_MATRIX: "15m" | |
| TEST_PG_BENCH_SCALES_MATRIX: "1" | |
| POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install | |
| DEFAULT_PG_VERSION: 16 | |
| TEST_OUTPUT: /tmp/test_output | |
| BUILD_TYPE: remote | |
| SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} | |
| PLATFORM: "neon-captest-pgvector" | |
| runs-on: [ self-hosted, us-east-2, x64 ] | |
| container: | |
| # The image has been uploaded to ECR manually: | |
| # docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:c6b377d375d8870e78a1ffb05ccdc14e101f4e7eb8841b275c23674fb66e088d \ | |
| # neondatabase/build-tools:c6b377d375d8870e78a1ffb05ccdc14e101f4e7eb8841b275c23674fb66e088d | |
| image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:c6b377d375d8870e78a1ffb05ccdc14e101f4e7eb8841b275c23674fb66e088d | |
| options: --init | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download Neon artifact | |
| uses: ./.github/actions/download | |
| with: | |
| name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact | |
| path: /tmp/neon/ | |
| prefix: 95038653d5f0fe416fbd51ab3296c44b8bf90e36/9703932965/1 | |
| - name: Add Postgres binaries to PATH | |
| run: | | |
| LD_LIBRARY_PATH="${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib" | |
| export LD_LIBRARY_PATH | |
| echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV | |
| ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version | |
| echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH | |
| - name: Set up Connection String | |
| id: set-up-connstr | |
| run: | | |
| CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }} | |
| echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT | |
| QUERIES=("SELECT version()") | |
| QUERIES+=("SHOW neon.tenant_id") | |
| QUERIES+=("SHOW neon.timeline_id") | |
| for q in "${QUERIES[@]}"; do | |
| psql ${CONNSTR} -c "${q}" | |
| done | |
| - name: Benchmark pgvector hnsw indexing | |
| uses: ./.github/actions/run-python-test-set | |
| with: | |
| build_type: ${{ env.BUILD_TYPE }} | |
| test_selection: performance/test_perf_olap.py | |
| run_in_parallel: false | |
| save_perf_report: ${{ env.SAVE_PERF_REPORT }} | |
| extra_params: -m remote_cluster --timeout 21600 -k test_pgvector_indexing | |
| env: | |
| VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" | |
| PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" | |
| BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} | |
| - name: Benchmark pgvector queries | |
| uses: ./.github/actions/run-python-test-set | |
| with: | |
| build_type: ${{ env.BUILD_TYPE }} | |
| test_selection: performance/test_perf_pgvector_queries.py | |
| run_in_parallel: false | |
| save_perf_report: ${{ env.SAVE_PERF_REPORT }} | |
| extra_params: -m remote_cluster --timeout 21600 | |
| env: | |
| BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} | |
| VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" | |
| PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" | |
| - name: Create Allure report | |
| if: ${{ !cancelled() }} | |
| uses: ./.github/actions/allure-report-generate | |
| - name: Post to a Slack channel | |
| if: ${{ github.event.schedule && failure() }} | |
| uses: slackapi/slack-github-action@v1 | |
| with: | |
| channel-id: "C033QLM5P7D" # dev-staging-stream | |
| slack-message: "Periodic perf testing neon-captest-pgvector: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} |