benchmarking.yml

name: Benchmarking

on:
  # uncomment to run on push for debugging your PR
  # push:
  #   branches: [ your branch ]
  schedule:
    # * is a special character in YAML so you have to quote this string
    #          ┌───────────── minute (0 - 59)
    #          │ ┌───────────── hour (0 - 23)
    #          │ │ ┌───────────── day of the month (1 - 31)
    #          │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
    #          │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    - cron:   '0 3 * * *' # run once a day, timezone is utc
  workflow_dispatch: # adds ability to run this manually
    inputs:
      region_id:
        description: 'Project region id. If not set, the default region will be used'
        required: false
        default: 'aws-us-east-2'
      save_perf_report:
        type: boolean
        description: 'Publish perf report. If not set, the report will be published only for the main branch'
        required: false
      collect_olap_explain:
        type: boolean
        description: 'Collect EXPLAIN ANALYZE for OLAP queries. If not set, EXPLAIN ANALYZE will not be collected'
        required: false
        default: false
      collect_pg_stat_statements:
        type: boolean
        description: 'Collect pg_stat_statements for OLAP queries. If not set, pg_stat_statements will not be collected'
        required: false
        default: false
      run_AWS_RDS_AND_AURORA:
        type: boolean
        description: 'AWS-RDS and AWS-AURORA normally only run on Saturday. Set this to true to run them on every workflow_dispatch'
        required: false
        default: false
      run_only_pgvector_tests:
        type: boolean
        description: 'Run pgvector tests but no other tests. If not set, all tests including pgvector tests will be run'
        required: false
        default: false
  pull_request:

defaults:
  run:
    shell: bash -euxo pipefail {0}

concurrency:
  # Allow only one workflow per any non-`main` branch.
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
  cancel-in-progress: true

env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }}

jobs:
  create-test-container:
    runs-on: [ self-hosted, small ]
    steps:
      - uses: actions/checkout@v4
      # Use custom DOCKER_CONFIG directory to avoid conflicts with default settings
      # The default value is ~/.docker
      - name: Set custom docker config directory
        run: |
          mkdir -p .docker-custom
          echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV

      - uses: docker/setup-buildx-action@v2

      - uses: docker/login-action@v3
        with:
          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

      - name: Download Neon artifact
        uses: ./.github/actions/download
        with:
          name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
          path: /tmp/neon/
          prefix: 0794b0a03498fe32c3445a401b19a6fb67da535a/9715565256/1

      - run: |
          mv /tmp/neon ./neon

          cat <<'EOF' > .dockerignore
            *
            !neon/
          EOF

          cat <<'EOF' > Dockerfile
            FROM neondatabase/build-tools:c5b33e139c1d8314370f29c76feaab6c14ec51f17ee1d9875c30c9077baf3844
            COPY neon /tmp/neon

            ENV LD_LIBRARY_PATH=/tmp/neon/pg_install/v16/lib:$LD_LIBRARY_PATH
            ENV PATH=/tmp/neon/pg_install/v16/bin:$PATH
          EOF

      - uses: docker/login-action@v3
        with:
          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

      - uses: docker/build-push-action@v5
        with:
          context: .
          provenance: false
          push: true
          pull: true
          file: Dockerfile
          tags: neondatabase/test:pr-8167-ssl32

      - name: Remove custom docker config directory
        if: always()
        run: |
          rm -rf .docker-custom

  pgbench-pgvector:
    needs: create-test-container
    env:
      TEST_PG_BENCH_DURATIONS_MATRIX: "15m"
      TEST_PG_BENCH_SCALES_MATRIX: "1"
      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
      DEFAULT_PG_VERSION: 16
      TEST_OUTPUT: /tmp/test_output
      BUILD_TYPE: remote
      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
      PLATFORM: "neon-captest-pgvector"

    runs-on: [ self-hosted, us-east-2, x64 ]
    container:
      # The image has been uploaded to ECR manually:
      # docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:c6b377d375d8870e78a1ffb05ccdc14e101f4e7eb8841b275c23674fb66e088d \
      #                                                                       neondatabase/build-tools:c6b377d375d8870e78a1ffb05ccdc14e101f4e7eb8841b275c23674fb66e088d
      image: neondatabase/test:pr-8167
      options: --init
      credentials:
        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

    steps:
    - uses: actions/checkout@v4

    # - name: Download Neon artifact
    #   uses: ./.github/actions/download
    #   with:
    #     name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
    #     path: /tmp/neon/
    #     prefix: 95038653d5f0fe416fbd51ab3296c44b8bf90e36/9703932965/1

    # - name: Add Postgres binaries to PATH
    #   run: |
    #     LD_LIBRARY_PATH="${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib"
    #     export LD_LIBRARY_PATH
    #     echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV

    #     ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
    #     echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH

    - name: Set up Connection String
      id: set-up-connstr
      run: |
        CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }}

        echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT

        QUERIES=("SELECT version()")
        QUERIES+=("SHOW neon.tenant_id")
        QUERIES+=("SHOW neon.timeline_id")

        for q in "${QUERIES[@]}"; do
          psql ${CONNSTR} -c "${q}"
        done

    # - name: Benchmark pgvector hnsw indexing
    #   uses: ./.github/actions/run-python-test-set
    #   with:
    #     build_type: ${{ env.BUILD_TYPE }}
    #     test_selection: performance/test_perf_olap.py
    #     run_in_parallel: false
    #     save_perf_report: ${{ env.SAVE_PERF_REPORT }}
    #     extra_params: -m remote_cluster --timeout 21600 -k test_pgvector_indexing
    #   env:
    #     VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
    #     PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
    #     BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}

    - name: Benchmark pgvector queries
      uses: ./.github/actions/run-python-test-set
      with:
        build_type: ${{ env.BUILD_TYPE }}
        test_selection: performance/test_perf_pgvector_queries.py
        run_in_parallel: false
        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
        extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_pgvector_halfvec
      env:
        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"

    - name: Create Allure report
      if: ${{ !cancelled() }}
      uses: ./.github/actions/allure-report-generate

    - name: Post to a Slack channel
      if: ${{ github.event.schedule && failure() }}
      uses: slackapi/slack-github-action@v1
      with:
        channel-id: "C033QLM5P7D" # dev-staging-stream
        slack-message: "Periodic perf testing neon-captest-pgvector: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
      env:
        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}