tests: turn on safekeeper eviction by default #23624
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build and Test | |
on: | |
push: | |
branches: | |
- main | |
- release | |
- release-proxy | |
pull_request: | |
defaults: | |
run: | |
shell: bash -euxo pipefail {0} | |
concurrency: | |
# Allow only one workflow per any non-`main` branch. | |
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} | |
cancel-in-progress: true | |
env: | |
RUST_BACKTRACE: 1 | |
COPT: '-Werror' | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} | |
# A concurrency group that we use for e2e-tests runs, matches `concurrency.group` above with `github.repository` as a prefix | |
E2E_CONCURRENCY_GROUP: ${{ github.repository }}-e2e-tests-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} | |
jobs: | |
check-permissions: | |
if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }} | |
uses: ./.github/workflows/check-permissions.yml | |
with: | |
github-event-name: ${{ github.event_name }} | |
cancel-previous-e2e-tests: | |
needs: [ check-permissions ] | |
if: github.event_name == 'pull_request' | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Cancel previous e2e-tests runs for this PR | |
env: | |
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} | |
run: | | |
gh workflow --repo neondatabase/cloud \ | |
run cancel-previous-in-concurrency-group.yml \ | |
--field concurrency_group="${{ env.E2E_CONCURRENCY_GROUP }}" | |
tag: | |
needs: [ check-permissions ] | |
runs-on: [ self-hosted, gen3, small ] | |
container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned | |
outputs: | |
build-tag: ${{steps.build-tag.outputs.tag}} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
- name: Get build tag | |
run: | | |
echo run:$GITHUB_RUN_ID | |
echo ref:$GITHUB_REF_NAME | |
echo rev:$(git rev-list --count HEAD) | |
if [[ "$GITHUB_REF_NAME" == "main" ]]; then | |
echo "tag=$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT | |
elif [[ "$GITHUB_REF_NAME" == "release" ]]; then | |
echo "tag=release-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT | |
elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then | |
echo "tag=release-proxy-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT | |
else | |
echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" | |
echo "tag=$GITHUB_RUN_ID" >> $GITHUB_OUTPUT | |
fi | |
shell: bash | |
id: build-tag | |
check-build-tools-image: | |
needs: [ check-permissions ] | |
uses: ./.github/workflows/check-build-tools-image.yml | |
build-build-tools-image: | |
needs: [ check-build-tools-image ] | |
uses: ./.github/workflows/build-build-tools-image.yml | |
with: | |
image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} | |
secrets: inherit | |
check-codestyle-python: | |
needs: [ check-permissions, build-build-tools-image ] | |
runs-on: [ self-hosted, gen3, small ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
options: --init | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: false | |
fetch-depth: 1 | |
- name: Cache poetry deps | |
uses: actions/cache@v4 | |
with: | |
path: ~/.cache/pypoetry/virtualenvs | |
key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-${{ hashFiles('poetry.lock') }} | |
- name: Install Python deps | |
run: ./scripts/pysync | |
- name: Run `ruff check` to ensure code format | |
run: poetry run ruff check . | |
- name: Run `ruff format` to ensure code format | |
run: poetry run ruff format --check . | |
- name: Run mypy to check types | |
run: poetry run mypy . | |
check-codestyle-rust: | |
needs: [ check-permissions, build-build-tools-image ] | |
runs-on: [ self-hosted, gen3, small ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
options: --init | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: true | |
fetch-depth: 1 | |
# Disabled for now | |
# - name: Restore cargo deps cache | |
# id: cache_cargo | |
# uses: actions/cache@v4 | |
# with: | |
# path: | | |
# !~/.cargo/registry/src | |
# ~/.cargo/git/ | |
# target/ | |
# key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-clippy-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} | |
# Some of our rust modules use FFI and need those to be checked | |
- name: Get postgres headers | |
run: make postgres-headers -j$(nproc) | |
# cargo hack runs the given cargo subcommand (clippy in this case) for all feature combinations. | |
# This will catch compiler & clippy warnings in all feature combinations. | |
# TODO: use cargo hack for build and test as well, but, that's quite expensive. | |
# NB: keep clippy args in sync with ./run_clippy.sh | |
- run: | | |
CLIPPY_COMMON_ARGS="$( source .neon_clippy_args; echo "$CLIPPY_COMMON_ARGS")" | |
if [ "$CLIPPY_COMMON_ARGS" = "" ]; then | |
echo "No clippy args found in .neon_clippy_args" | |
exit 1 | |
fi | |
echo "CLIPPY_COMMON_ARGS=${CLIPPY_COMMON_ARGS}" >> $GITHUB_ENV | |
- name: Run cargo clippy (debug) | |
run: cargo hack --feature-powerset clippy $CLIPPY_COMMON_ARGS | |
- name: Run cargo clippy (release) | |
run: cargo hack --feature-powerset clippy --release $CLIPPY_COMMON_ARGS | |
- name: Check documentation generation | |
run: cargo doc --workspace --no-deps --document-private-items | |
env: | |
RUSTDOCFLAGS: "-Dwarnings -Arustdoc::private_intra_doc_links" | |
# Use `${{ !cancelled() }}` to run quck tests after the longer clippy run | |
- name: Check formatting | |
if: ${{ !cancelled() }} | |
run: cargo fmt --all -- --check | |
# https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci | |
- name: Check rust dependencies | |
if: ${{ !cancelled() }} | |
run: | | |
cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date | |
cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack | |
# https://github.com/EmbarkStudios/cargo-deny | |
- name: Check rust licenses/bans/advisories/sources | |
if: ${{ !cancelled() }} | |
run: cargo deny check --hide-inclusion-graph | |
build-neon: | |
needs: [ check-permissions, tag, build-build-tools-image ] | |
runs-on: [ self-hosted, gen3, large ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
# Raise locked memory limit for tokio-epoll-uring. | |
# On 5.10 LTS kernels < 5.10.162 (and generally mainline kernels < 5.12), | |
# io_uring will account the memory of the CQ and SQ as locked. | |
# More details: https://github.com/neondatabase/neon/issues/6373#issuecomment-1905814391 | |
options: --init --shm-size=512mb --ulimit memlock=67108864:67108864 | |
strategy: | |
fail-fast: false | |
matrix: | |
build_type: [ debug, release ] | |
env: | |
BUILD_TYPE: ${{ matrix.build_type }} | |
GIT_VERSION: ${{ github.event.pull_request.head.sha || github.sha }} | |
BUILD_TAG: ${{ needs.tag.outputs.build-tag }} | |
steps: | |
- name: Fix git ownership | |
run: | | |
# Workaround for `fatal: detected dubious ownership in repository at ...` | |
# | |
# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers | |
# Ref https://github.com/actions/checkout/issues/785 | |
# | |
git config --global --add safe.directory ${{ github.workspace }} | |
git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
for r in 14 15 16; do | |
git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r" | |
git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r" | |
done | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: true | |
fetch-depth: 1 | |
- name: Set pg 14 revision for caching | |
id: pg_v14_rev | |
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT | |
- name: Set pg 15 revision for caching | |
id: pg_v15_rev | |
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT | |
- name: Set pg 16 revision for caching | |
id: pg_v16_rev | |
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT | |
# Set some environment variables used by all the steps. | |
# | |
# CARGO_FLAGS is extra options to pass to "cargo build", "cargo test" etc. | |
# It also includes --features, if any | |
# | |
# CARGO_FEATURES is passed to "cargo metadata". It is separate from CARGO_FLAGS, | |
# because "cargo metadata" doesn't accept --release or --debug options | |
# | |
# We run tests with addtional features, that are turned off by default (e.g. in release builds), see | |
# corresponding Cargo.toml files for their descriptions. | |
- name: Set env variables | |
run: | | |
CARGO_FEATURES="--features testing" | |
if [[ $BUILD_TYPE == "debug" ]]; then | |
cov_prefix="scripts/coverage --profraw-prefix=$GITHUB_JOB --dir=/tmp/coverage run" | |
CARGO_FLAGS="--locked" | |
elif [[ $BUILD_TYPE == "release" ]]; then | |
cov_prefix="" | |
CARGO_FLAGS="--locked --release" | |
fi | |
{ | |
echo "cov_prefix=${cov_prefix}" | |
echo "CARGO_FEATURES=${CARGO_FEATURES}" | |
echo "CARGO_FLAGS=${CARGO_FLAGS}" | |
echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo" | |
} >> $GITHUB_ENV | |
# Disabled for now | |
# Don't include the ~/.cargo/registry/src directory. It contains just | |
# uncompressed versions of the crates in ~/.cargo/registry/cache | |
# directory, and it's faster to let 'cargo' to rebuild it from the | |
# compressed crates. | |
# - name: Cache cargo deps | |
# id: cache_cargo | |
# uses: actions/cache@v4 | |
# with: | |
# path: | | |
# ~/.cargo/registry/ | |
# !~/.cargo/registry/src | |
# ~/.cargo/git/ | |
# target/ | |
# # Fall back to older versions of the key, if no cache for current Cargo.lock was found | |
# key: | | |
# v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} | |
# v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}- | |
- name: Cache postgres v14 build | |
id: cache_pg_14 | |
uses: actions/cache@v4 | |
with: | |
path: pg_install/v14 | |
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }} | |
- name: Cache postgres v15 build | |
id: cache_pg_15 | |
uses: actions/cache@v4 | |
with: | |
path: pg_install/v15 | |
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }} | |
- name: Cache postgres v16 build | |
id: cache_pg_16 | |
uses: actions/cache@v4 | |
with: | |
path: pg_install/v16 | |
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile', 'Dockerfile.build-tools') }} | |
- name: Build postgres v14 | |
if: steps.cache_pg_14.outputs.cache-hit != 'true' | |
run: mold -run make postgres-v14 -j$(nproc) | |
- name: Build postgres v15 | |
if: steps.cache_pg_15.outputs.cache-hit != 'true' | |
run: mold -run make postgres-v15 -j$(nproc) | |
- name: Build postgres v16 | |
if: steps.cache_pg_16.outputs.cache-hit != 'true' | |
run: mold -run make postgres-v16 -j$(nproc) | |
- name: Build neon extensions | |
run: mold -run make neon-pg-ext -j$(nproc) | |
- name: Build walproposer-lib | |
run: mold -run make walproposer-lib -j$(nproc) | |
- name: Run cargo build | |
run: | | |
PQ_LIB_DIR=$(pwd)/pg_install/v16/lib | |
export PQ_LIB_DIR | |
${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests | |
# Do install *before* running rust tests because they might recompile the | |
# binaries with different features/flags. | |
- name: Install rust binaries | |
run: | | |
# Install target binaries | |
mkdir -p /tmp/neon/bin/ | |
binaries=$( | |
${cov_prefix} cargo metadata $CARGO_FEATURES --format-version=1 --no-deps | | |
jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name' | |
) | |
for bin in $binaries; do | |
SRC=target/$BUILD_TYPE/$bin | |
DST=/tmp/neon/bin/$bin | |
cp "$SRC" "$DST" | |
done | |
# Install test executables and write list of all binaries (for code coverage) | |
if [[ $BUILD_TYPE == "debug" ]]; then | |
# Keep bloated coverage data files away from the rest of the artifact | |
mkdir -p /tmp/coverage/ | |
mkdir -p /tmp/neon/test_bin/ | |
test_exe_paths=$( | |
${cov_prefix} cargo test $CARGO_FLAGS $CARGO_FEATURES --message-format=json --no-run | | |
jq -r '.executable | select(. != null)' | |
) | |
for bin in $test_exe_paths; do | |
SRC=$bin | |
DST=/tmp/neon/test_bin/$(basename $bin) | |
# We don't need debug symbols for code coverage, so strip them out to make | |
# the artifact smaller. | |
strip "$SRC" -o "$DST" | |
echo "$DST" >> /tmp/coverage/binaries.list | |
done | |
for bin in $binaries; do | |
echo "/tmp/neon/bin/$bin" >> /tmp/coverage/binaries.list | |
done | |
fi | |
- name: Run rust tests | |
env: | |
NEXTEST_RETRIES: 3 | |
run: | | |
PQ_LIB_DIR=$(pwd)/pg_install/v16/lib | |
export PQ_LIB_DIR | |
LD_LIBRARY_PATH=$(pwd)/pg_install/v16/lib | |
export LD_LIBRARY_PATH | |
#nextest does not yet support running doctests | |
cargo test --doc $CARGO_FLAGS $CARGO_FEATURES | |
for io_engine in std-fs tokio-epoll-uring ; do | |
NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=$io_engine ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES | |
done | |
# Run separate tests for real S3 | |
export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty | |
export REMOTE_STORAGE_S3_BUCKET=neon-github-ci-tests | |
export REMOTE_STORAGE_S3_REGION=eu-central-1 | |
${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_s3)' | |
# Run separate tests for real Azure Blob Storage | |
# XXX: replace region with `eu-central-1`-like region | |
export ENABLE_REAL_AZURE_REMOTE_STORAGE=y | |
export AZURE_STORAGE_ACCOUNT="${{ secrets.AZURE_STORAGE_ACCOUNT_DEV }}" | |
export AZURE_STORAGE_ACCESS_KEY="${{ secrets.AZURE_STORAGE_ACCESS_KEY_DEV }}" | |
export REMOTE_STORAGE_AZURE_CONTAINER="${{ vars.REMOTE_STORAGE_AZURE_CONTAINER }}" | |
export REMOTE_STORAGE_AZURE_REGION="${{ vars.REMOTE_STORAGE_AZURE_REGION }}" | |
${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E 'package(remote_storage)' -E 'test(test_real_azure)' | |
- name: Install postgres binaries | |
run: cp -a pg_install /tmp/neon/pg_install | |
- name: Upload Neon artifact | |
uses: ./.github/actions/upload | |
with: | |
name: neon-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-artifact | |
path: /tmp/neon | |
# XXX: keep this after the binaries.list is formed, so the coverage can properly work later | |
- name: Merge and upload coverage data | |
if: matrix.build_type == 'debug' | |
uses: ./.github/actions/save-coverage-data | |
regress-tests: | |
needs: [ check-permissions, build-neon, build-build-tools-image, tag ] | |
runs-on: [ self-hosted, gen3, large ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
# for changed limits, see comments on `options:` earlier in this file | |
options: --init --shm-size=512mb --ulimit memlock=67108864:67108864 | |
strategy: | |
fail-fast: false | |
matrix: | |
build_type: [ debug, release ] | |
pg_version: [ v14, v15, v16 ] | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: true | |
fetch-depth: 1 | |
- name: Pytest regression tests | |
uses: ./.github/actions/run-python-test-set | |
timeout-minutes: 60 | |
with: | |
build_type: ${{ matrix.build_type }} | |
test_selection: regress | |
needs_postgres_source: true | |
run_with_real_s3: true | |
real_s3_bucket: neon-github-ci-tests | |
real_s3_region: eu-central-1 | |
rerun_flaky: true | |
pg_version: ${{ matrix.pg_version }} | |
env: | |
TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} | |
CHECK_ONDISK_DATA_COMPATIBILITY: nonempty | |
BUILD_TAG: ${{ needs.tag.outputs.build-tag }} | |
PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring | |
PAGESERVER_GET_VECTORED_IMPL: vectored | |
PAGESERVER_GET_IMPL: vectored | |
PAGESERVER_VALIDATE_VEC_GET: true | |
# Temporary disable this step until we figure out why it's so flaky | |
# Ref https://github.com/neondatabase/neon/issues/4540 | |
- name: Merge and upload coverage data | |
if: | | |
false && | |
matrix.build_type == 'debug' && matrix.pg_version == 'v14' | |
uses: ./.github/actions/save-coverage-data | |
get-benchmarks-durations: | |
outputs: | |
json: ${{ steps.get-benchmark-durations.outputs.json }} | |
needs: [ check-permissions, build-build-tools-image ] | |
runs-on: [ self-hosted, gen3, small ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
options: --init | |
if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Cache poetry deps | |
uses: actions/cache@v4 | |
with: | |
path: ~/.cache/pypoetry/virtualenvs | |
key: v1-${{ runner.os }}-${{ runner.arch }}-python-deps-${{ hashFiles('poetry.lock') }} | |
- name: Install Python deps | |
run: ./scripts/pysync | |
- name: get benchmark durations | |
id: get-benchmark-durations | |
env: | |
TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} | |
run: | | |
poetry run ./scripts/benchmark_durations.py "${TEST_RESULT_CONNSTR}" \ | |
--days 10 \ | |
--output /tmp/benchmark_durations.json | |
echo "json=$(jq --compact-output '.' /tmp/benchmark_durations.json)" >> $GITHUB_OUTPUT | |
benchmarks: | |
needs: [ check-permissions, build-neon, build-build-tools-image, get-benchmarks-durations ] | |
runs-on: [ self-hosted, gen3, small ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
# for changed limits, see comments on `options:` earlier in this file | |
options: --init --shm-size=512mb --ulimit memlock=67108864:67108864 | |
if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') | |
strategy: | |
fail-fast: false | |
matrix: | |
# the amount of groups (N) should be reflected in `extra_params: --splits N ...` | |
pytest_split_group: [ 1, 2, 3, 4, 5 ] | |
build_type: [ release ] | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
- name: Pytest benchmarks | |
uses: ./.github/actions/run-python-test-set | |
with: | |
build_type: ${{ matrix.build_type }} | |
test_selection: performance | |
run_in_parallel: false | |
save_perf_report: ${{ github.ref_name == 'main' }} | |
extra_params: --splits 5 --group ${{ matrix.pytest_split_group }} | |
benchmark_durations: ${{ needs.get-benchmarks-durations.outputs.json }} | |
env: | |
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" | |
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" | |
TEST_RESULT_CONNSTR: "${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}" | |
PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring | |
PAGESERVER_GET_VECTORED_IMPL: vectored | |
PAGESERVER_GET_IMPL: vectored | |
PAGESERVER_VALIDATE_VEC_GET: false | |
# XXX: no coverage data handling here, since benchmarks are run on release builds, | |
# while coverage is currently collected for the debug ones | |
report-benchmarks-failures: | |
needs: [ benchmarks, create-test-report ] | |
if: github.ref_name == 'main' && failure() && needs.benchmarks.result == 'failure' | |
runs-on: ubuntu-22.04 | |
steps: | |
- uses: slackapi/slack-github-action@v1 | |
with: | |
channel-id: C060CNA47S9 # on-call-staging-storage-stream | |
slack-message: | | |
Benchmarks failed on main: ${{ github.event.head_commit.url }} | |
Allure report: ${{ needs.create-test-report.outputs.report-url }} | |
env: | |
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
create-test-report: | |
needs: [ check-permissions, regress-tests, coverage-report, benchmarks, build-build-tools-image ] | |
if: ${{ !cancelled() && contains(fromJSON('["skipped", "success"]'), needs.check-permissions.result) }} | |
outputs: | |
report-url: ${{ steps.create-allure-report.outputs.report-url }} | |
runs-on: [ self-hosted, gen3, small ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
options: --init | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Create Allure report | |
if: ${{ !cancelled() }} | |
id: create-allure-report | |
uses: ./.github/actions/allure-report-generate | |
with: | |
store-test-results-into-db: true | |
env: | |
REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} | |
- uses: actions/github-script@v7 | |
if: ${{ !cancelled() }} | |
with: | |
# Retry script for 5XX server errors: https://github.com/actions/github-script#retries | |
retries: 5 | |
script: | | |
const report = { | |
reportUrl: "${{ steps.create-allure-report.outputs.report-url }}", | |
reportJsonUrl: "${{ steps.create-allure-report.outputs.report-json-url }}", | |
} | |
const coverage = { | |
coverageUrl: "${{ needs.coverage-report.outputs.coverage-html }}", | |
summaryJsonUrl: "${{ needs.coverage-report.outputs.coverage-json }}", | |
} | |
const script = require("./scripts/comment-test-report.js") | |
await script({ | |
github, | |
context, | |
fetch, | |
report, | |
coverage, | |
}) | |
coverage-report: | |
needs: [ check-permissions, regress-tests, build-build-tools-image ] | |
runs-on: [ self-hosted, gen3, small ] | |
container: | |
image: ${{ needs.build-build-tools-image.outputs.image }} | |
credentials: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
options: --init | |
strategy: | |
fail-fast: false | |
matrix: | |
build_type: [ debug ] | |
outputs: | |
coverage-html: ${{ steps.upload-coverage-report-new.outputs.report-url }} | |
coverage-json: ${{ steps.upload-coverage-report-new.outputs.summary-json }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: true | |
fetch-depth: 0 | |
- name: Get Neon artifact | |
uses: ./.github/actions/download | |
with: | |
name: neon-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-artifact | |
path: /tmp/neon | |
- name: Get coverage artifact | |
uses: ./.github/actions/download | |
with: | |
name: coverage-data-artifact | |
path: /tmp/coverage | |
- name: Merge coverage data | |
run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge | |
- name: Build coverage report | |
env: | |
COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.event.pull_request.head.sha || github.sha }} | |
run: | | |
scripts/coverage --dir=/tmp/coverage \ | |
report \ | |
--input-objects=/tmp/coverage/binaries.list \ | |
--commit-url=${COMMIT_URL} \ | |
--format=github | |
scripts/coverage --dir=/tmp/coverage \ | |
report \ | |
--input-objects=/tmp/coverage/binaries.list \ | |
--format=lcov | |
- name: Build coverage report NEW | |
id: upload-coverage-report-new | |
env: | |
BUCKET: neon-github-public-dev | |
# A differential coverage report is available only for PRs. | |
# (i.e. for pushes into main/release branches we have a regular coverage report) | |
COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
BASE_SHA: ${{ github.event.pull_request.base.sha || github.sha }} | |
run: | | |
CURRENT="${COMMIT_SHA}" | |
BASELINE="$(git merge-base $BASE_SHA $CURRENT)" | |
cp /tmp/coverage/report/lcov.info ./${CURRENT}.info | |
GENHTML_ARGS="--ignore-errors path,unmapped,empty --synthesize-missing --demangle-cpp rustfilt --output-directory lcov-html ${CURRENT}.info" | |
# Use differential coverage if the baseline coverage exists. | |
# It can be missing if the coverage repoer wasn't uploaded yet or tests has failed on BASELINE commit. | |
if aws s3 cp --only-show-errors s3://${BUCKET}/code-coverage/${BASELINE}/lcov.info ./${BASELINE}.info; then | |
git diff ${BASELINE} ${CURRENT} -- '*.rs' > baseline-current.diff | |
GENHTML_ARGS="--baseline-file ${BASELINE}.info --diff-file baseline-current.diff ${GENHTML_ARGS}" | |
fi | |
genhtml ${GENHTML_ARGS} | |
aws s3 cp --only-show-errors --recursive ./lcov-html/ s3://${BUCKET}/code-coverage/${COMMIT_SHA}/lcov | |
REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/lcov/index.html | |
echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT | |
REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/lcov/summary.json | |
echo "summary-json=${REPORT_URL}" >> $GITHUB_OUTPUT | |
- uses: actions/github-script@v7 | |
env: | |
REPORT_URL_NEW: ${{ steps.upload-coverage-report-new.outputs.report-url }} | |
COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
with: | |
script: | | |
const { REPORT_URL_NEW, COMMIT_SHA } = process.env | |
await github.rest.repos.createCommitStatus({ | |
owner: context.repo.owner, | |
repo: context.repo.repo, | |
sha: `${COMMIT_SHA}`, | |
state: 'success', | |
target_url: `${REPORT_URL_NEW}`, | |
context: 'Code coverage report NEW', | |
}) | |
trigger-e2e-tests: | |
if: ${{ !github.event.pull_request.draft || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') || github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' }} | |
needs: [ check-permissions, promote-images, tag ] | |
uses: ./.github/workflows/trigger-e2e-tests.yml | |
secrets: inherit | |
neon-image-arch: | |
needs: [ check-permissions, build-build-tools-image, tag ] | |
strategy: | |
matrix: | |
arch: [ x64, arm64 ] | |
runs-on: ${{ fromJson(format('["self-hosted", "gen3", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: true | |
fetch-depth: 0 | |
# Use custom DOCKER_CONFIG directory to avoid conflicts with default settings | |
# The default value is ~/.docker | |
- name: Set custom docker config directory | |
run: | | |
mkdir -p .docker-custom | |
echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV | |
- uses: docker/setup-buildx-action@v3 | |
with: | |
cache-binary: false | |
- uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
- uses: docker/build-push-action@v6 | |
with: | |
context: . | |
build-args: | | |
GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} | |
BUILD_TAG=${{ needs.tag.outputs.build-tag }} | |
TAG=${{ needs.build-build-tools-image.outputs.image-tag }} | |
provenance: false | |
push: true | |
pull: true | |
file: Dockerfile | |
cache-from: type=registry,ref=neondatabase/neon:cache-${{ matrix.arch }} | |
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=neondatabase/neon:cache-{0},mode=max', matrix.arch) || '' }} | |
tags: | | |
neondatabase/neon:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }} | |
- name: Remove custom docker config directory | |
if: always() | |
run: | | |
rm -rf .docker-custom | |
neon-image: | |
needs: [ neon-image-arch, tag ] | |
runs-on: ubuntu-22.04 | |
steps: | |
- uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
- name: Create multi-arch image | |
run: | | |
docker buildx imagetools create -t neondatabase/neon:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/neon:${{ needs.tag.outputs.build-tag }}-x64 \ | |
neondatabase/neon:${{ needs.tag.outputs.build-tag }}-arm64 | |
- uses: docker/login-action@v3 | |
with: | |
registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com | |
username: ${{ secrets.AWS_ACCESS_KEY_DEV }} | |
password: ${{ secrets.AWS_SECRET_KEY_DEV }} | |
- name: Push multi-arch image to ECR | |
run: | | |
docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/neon:${{ needs.tag.outputs.build-tag }} | |
compute-node-image-arch: | |
needs: [ check-permissions, build-build-tools-image, tag ] | |
strategy: | |
fail-fast: false | |
matrix: | |
version: [ v14, v15, v16 ] | |
arch: [ x64, arm64 ] | |
runs-on: ${{ fromJson(format('["self-hosted", "gen3", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: true | |
fetch-depth: 0 | |
# Use custom DOCKER_CONFIG directory to avoid conflicts with default settings | |
# The default value is ~/.docker | |
- name: Set custom docker config directory | |
run: | | |
mkdir -p .docker-custom | |
echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV | |
- uses: docker/setup-buildx-action@v3 | |
with: | |
cache-binary: false | |
# Disable parallelism for docker buildkit. | |
# As we already build everything with `make -j$(nproc)`, running it in additional level of parallelisam blows up the Runner. | |
buildkitd-config-inline: | | |
[worker.oci] | |
max-parallelism = 1 | |
- uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
- uses: docker/login-action@v3 | |
with: | |
registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com | |
username: ${{ secrets.AWS_ACCESS_KEY_DEV }} | |
password: ${{ secrets.AWS_SECRET_KEY_DEV }} | |
- name: Build compute-node image | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
build-args: | | |
GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} | |
PG_VERSION=${{ matrix.version }} | |
BUILD_TAG=${{ needs.tag.outputs.build-tag }} | |
TAG=${{ needs.build-build-tools-image.outputs.image-tag }} | |
provenance: false | |
push: true | |
pull: true | |
file: Dockerfile.compute-node | |
cache-from: type=registry,ref=neondatabase/compute-node-${{ matrix.version }}:cache-${{ matrix.arch }} | |
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=neondatabase/compute-node-{0}:cache-{1},mode=max', matrix.version, matrix.arch) || '' }} | |
tags: | | |
neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }} | |
- name: Build neon extensions test image | |
if: matrix.version == 'v16' | |
uses: docker/build-push-action@v6 | |
with: | |
context: . | |
build-args: | | |
GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} | |
PG_VERSION=${{ matrix.version }} | |
BUILD_TAG=${{ needs.tag.outputs.build-tag }} | |
TAG=${{ needs.build-build-tools-image.outputs.image-tag }} | |
provenance: false | |
push: true | |
pull: true | |
file: Dockerfile.compute-node | |
target: neon-pg-ext-test | |
cache-from: type=registry,ref=neondatabase/neon-test-extensions-${{ matrix.version }}:cache-${{ matrix.arch }} | |
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=neondatabase/neon-test-extensions-{0}:cache-{1},mode=max', matrix.version, matrix.arch) || '' }} | |
tags: | | |
neondatabase/neon-test-extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}-${{ matrix.arch }} | |
- name: Build compute-tools image | |
# compute-tools are Postgres independent, so build it only once | |
if: matrix.version == 'v16' | |
uses: docker/build-push-action@v6 | |
with: | |
target: compute-tools-image | |
context: . | |
build-args: | | |
GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} | |
BUILD_TAG=${{ needs.tag.outputs.build-tag }} | |
TAG=${{ needs.build-build-tools-image.outputs.image-tag }} | |
provenance: false | |
push: true | |
pull: true | |
file: Dockerfile.compute-node | |
tags: | | |
neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }} | |
- name: Remove custom docker config directory | |
if: always() | |
run: | | |
rm -rf .docker-custom | |
compute-node-image: | |
needs: [ compute-node-image-arch, tag ] | |
runs-on: ubuntu-22.04 | |
strategy: | |
matrix: | |
version: [ v14, v15, v16 ] | |
steps: | |
- uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
- name: Create multi-arch compute-node image | |
run: | | |
docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-x64 \ | |
neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-arm64 | |
- name: Create multi-arch neon-test-extensions image | |
if: matrix.version == 'v16' | |
run: | | |
docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/neon-test-extensions-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-x64 \ | |
neondatabase/neon-test-extensions-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-arm64 | |
- name: Create multi-arch compute-tools image | |
if: matrix.version == 'v16' | |
run: | | |
docker buildx imagetools create -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-x64 \ | |
neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-arm64 | |
- uses: docker/login-action@v3 | |
with: | |
registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com | |
username: ${{ secrets.AWS_ACCESS_KEY_DEV }} | |
password: ${{ secrets.AWS_SECRET_KEY_DEV }} | |
- name: Push multi-arch compute-node-${{ matrix.version }} image to ECR | |
run: | | |
docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} | |
- name: Push multi-arch compute-tools image to ECR | |
if: matrix.version == 'v16' | |
run: | | |
docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} | |
vm-compute-node-image: | |
needs: [ check-permissions, tag, compute-node-image ] | |
runs-on: [ self-hosted, gen3, large ] | |
strategy: | |
fail-fast: false | |
matrix: | |
version: [ v14, v15, v16 ] | |
env: | |
VM_BUILDER_VERSION: v0.29.3 | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
- name: Downloading vm-builder | |
run: | | |
curl -fL https://github.com/neondatabase/autoscaling/releases/download/$VM_BUILDER_VERSION/vm-builder -o vm-builder | |
chmod +x vm-builder | |
# Use custom DOCKER_CONFIG directory to avoid conflicts with default settings | |
# The default value is ~/.docker | |
- name: Set custom docker config directory | |
run: | | |
mkdir -p .docker-custom | |
echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV | |
- uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
# Note: we need a separate pull step here because otherwise vm-builder will try to pull, and | |
# it won't have the proper authentication (written at v0.6.0) | |
- name: Pulling compute-node image | |
run: | | |
docker pull neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} | |
- name: Build vm image | |
run: | | |
./vm-builder \ | |
-spec=vm-image-spec.yaml \ | |
-src=neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} \ | |
-dst=neondatabase/vm-compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} | |
- name: Pushing vm-compute-node image | |
run: | | |
docker push neondatabase/vm-compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} | |
- name: Remove custom docker config directory | |
if: always() | |
run: | | |
rm -rf .docker-custom | |
test-images: | |
needs: [ check-permissions, tag, neon-image, compute-node-image ] | |
strategy: | |
fail-fast: false | |
matrix: | |
arch: [ x64, arm64 ] | |
runs-on: ${{ fromJson(format('["self-hosted", "gen3", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }} | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
# Use custom DOCKER_CONFIG directory to avoid conflicts with default settings | |
# The default value is ~/.docker | |
- name: Set custom docker config directory | |
run: | | |
mkdir -p .docker-custom | |
echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV | |
- uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
# `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library. | |
# Pick pageserver as currently the only binary with extra "version" features printed in the string to verify. | |
# Regular pageserver version string looks like | |
# Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: [] | |
# Bad versions might loop like: | |
# Neon page server git-env:local failpoints: true, features: ["testing"] | |
# Ensure that we don't have bad versions. | |
- name: Verify image versions | |
shell: bash # ensure no set -e for better error messages | |
run: | | |
pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.tag.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") | |
echo "Pageserver version string: $pageserver_version" | |
if ! echo "$pageserver_version" | grep -qv 'git-env:local' ; then | |
echo "Pageserver version should not be the default Dockerfile one" | |
exit 1 | |
fi | |
if ! echo "$pageserver_version" | grep -qv '"testing"' ; then | |
echo "Pageserver version should have no testing feature enabled" | |
exit 1 | |
fi | |
- name: Verify docker-compose example and test extensions | |
timeout-minutes: 20 | |
run: env TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh | |
- name: Print logs and clean up | |
if: always() | |
run: | | |
docker compose -f ./docker-compose/docker-compose.yml logs || 0 | |
docker compose -f ./docker-compose/docker-compose.yml down | |
- name: Remove custom docker config directory | |
if: always() | |
run: | | |
rm -rf .docker-custom | |
promote-images: | |
needs: [ check-permissions, tag, test-images, vm-compute-node-image ] | |
runs-on: ubuntu-22.04 | |
env: | |
VERSIONS: v14 v15 v16 | |
steps: | |
- uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} | |
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} | |
- name: Login to dev ECR | |
uses: docker/login-action@v3 | |
with: | |
registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com | |
username: ${{ secrets.AWS_ACCESS_KEY_DEV }} | |
password: ${{ secrets.AWS_SECRET_KEY_DEV }} | |
- name: Copy vm-compute-node images to ECR | |
run: | | |
for version in ${VERSIONS}; do | |
docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} \ | |
neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} | |
done | |
- name: Add latest tag to images | |
if: github.ref_name == 'main' | |
run: | | |
for repo in neondatabase 369495373322.dkr.ecr.eu-central-1.amazonaws.com; do | |
docker buildx imagetools create -t $repo/neon:latest \ | |
$repo/neon:${{ needs.tag.outputs.build-tag }} | |
docker buildx imagetools create -t $repo/compute-tools:latest \ | |
$repo/compute-tools:${{ needs.tag.outputs.build-tag }} | |
for version in ${VERSIONS}; do | |
docker buildx imagetools create -t $repo/compute-node-${version}:latest \ | |
$repo/compute-node-${version}:${{ needs.tag.outputs.build-tag }} | |
docker buildx imagetools create -t $repo/vm-compute-node-${version}:latest \ | |
$repo/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} | |
done | |
done | |
docker buildx imagetools create -t neondatabase/neon-test-extensions-v16:latest \ | |
neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }} | |
- name: Login to prod ECR | |
uses: docker/login-action@v3 | |
if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' | |
with: | |
registry: 093970136003.dkr.ecr.eu-central-1.amazonaws.com | |
username: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_ACCESS_KEY_ID }} | |
password: ${{ secrets.PROD_GHA_RUNNER_LIMITED_AWS_SECRET_ACCESS_KEY }} | |
- name: Copy all images to prod ECR | |
if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' | |
run: | | |
for image in neon compute-tools {vm-,}compute-node-{v14,v15,v16}; do | |
docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} \ | |
369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} | |
done | |
trigger-custom-extensions-build-and-wait: | |
needs: [ check-permissions, tag ] | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Set PR's status to pending and request a remote CI test | |
run: | | |
COMMIT_SHA=${{ github.event.pull_request.head.sha || github.sha }} | |
REMOTE_REPO="${{ github.repository_owner }}/build-custom-extensions" | |
curl -f -X POST \ | |
https://github.com/gitapi/repos/${{ github.repository }}/statuses/$COMMIT_SHA \ | |
-H "Accept: application/vnd.github.v3+json" \ | |
--user "${{ secrets.CI_ACCESS_TOKEN }}" \ | |
--data \ | |
"{ | |
\"state\": \"pending\", | |
\"context\": \"build-and-upload-extensions\", | |
\"description\": \"[$REMOTE_REPO] Remote CI job is about to start\" | |
}" | |
curl -f -X POST \ | |
https://github.com/gitapi/repos/$REMOTE_REPO/actions/workflows/build_and_upload_extensions.yml/dispatches \ | |
-H "Accept: application/vnd.github.v3+json" \ | |
--user "${{ secrets.CI_ACCESS_TOKEN }}" \ | |
--data \ | |
"{ | |
\"ref\": \"main\", | |
\"inputs\": { | |
\"ci_job_name\": \"build-and-upload-extensions\", | |
\"commit_hash\": \"$COMMIT_SHA\", | |
\"remote_repo\": \"${{ github.repository }}\", | |
\"compute_image_tag\": \"${{ needs.tag.outputs.build-tag }}\", | |
\"remote_branch_name\": \"${{ github.ref_name }}\" | |
} | |
}" | |
- name: Wait for extension build to finish | |
env: | |
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} | |
run: | | |
TIMEOUT=1800 # 30 minutes, usually it takes ~2-3 minutes, but if runners are busy, it might take longer | |
INTERVAL=15 # try each N seconds | |
last_status="" # a variable to carry the last status of the "build-and-upload-extensions" context | |
for ((i=0; i <= TIMEOUT; i+=INTERVAL)); do | |
sleep $INTERVAL | |
# Get statuses for the latest commit in the PR / branch | |
gh api \ | |
-H "Accept: application/vnd.github+json" \ | |
-H "X-GitHub-Api-Version: 2022-11-28" \ | |
"/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha || github.sha }}" > statuses.json | |
# Get the latest status for the "build-and-upload-extensions" context | |
last_status=$(jq --raw-output '[.[] | select(.context == "build-and-upload-extensions")] | sort_by(.created_at)[-1].state' statuses.json) | |
if [ "${last_status}" = "pending" ]; then | |
# Extension build is still in progress. | |
continue | |
elif [ "${last_status}" = "success" ]; then | |
# Extension build is successful. | |
exit 0 | |
else | |
# Status is neither "pending" nor "success", exit the loop and fail the job. | |
break | |
fi | |
done | |
# Extension build failed, print `statuses.json` for debugging and fail the job. | |
jq '.' statuses.json | |
echo >&2 "Status of extension build is '${last_status}' != 'success'" | |
exit 1 | |
deploy: | |
needs: [ check-permissions, promote-images, tag, regress-tests, trigger-custom-extensions-build-and-wait ] | |
if: github.ref_name == 'main' || github.ref_name == 'release'|| github.ref_name == 'release-proxy' | |
runs-on: [ self-hosted, gen3, small ] | |
container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest | |
steps: | |
- name: Fix git ownership | |
run: | | |
# Workaround for `fatal: detected dubious ownership in repository at ...` | |
# | |
# Use both ${{ github.workspace }} and ${GITHUB_WORKSPACE} because they're different on host and in containers | |
# Ref https://github.com/actions/checkout/issues/785 | |
# | |
git config --global --add safe.directory ${{ github.workspace }} | |
git config --global --add safe.directory ${GITHUB_WORKSPACE} | |
for r in 14 15 16; do | |
git config --global --add safe.directory "${{ github.workspace }}/vendor/postgres-v$r" | |
git config --global --add safe.directory "${GITHUB_WORKSPACE}/vendor/postgres-v$r" | |
done | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
submodules: false | |
fetch-depth: 0 | |
- name: Trigger deploy workflow | |
env: | |
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} | |
run: | | |
if [[ "$GITHUB_REF_NAME" == "main" ]]; then | |
gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main -f branch=main -f dockerTag=${{needs.tag.outputs.build-tag}} -f deployPreprodRegion=false | |
gh workflow --repo neondatabase/azure run deploy.yml -f dockerTag=${{needs.tag.outputs.build-tag}} | |
elif [[ "$GITHUB_REF_NAME" == "release" ]]; then | |
gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main \ | |
-f deployPgSniRouter=false \ | |
-f deployProxy=false \ | |
-f deployStorage=true \ | |
-f deployStorageBroker=true \ | |
-f deployStorageController=true \ | |
-f branch=main \ | |
-f dockerTag=${{needs.tag.outputs.build-tag}} \ | |
-f deployPreprodRegion=true | |
gh workflow --repo neondatabase/aws run deploy-prod.yml --ref main \ | |
-f deployStorage=true \ | |
-f deployStorageBroker=true \ | |
-f deployStorageController=true \ | |
-f branch=main \ | |
-f dockerTag=${{needs.tag.outputs.build-tag}} | |
elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then | |
gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main \ | |
-f deployPgSniRouter=true \ | |
-f deployProxy=true \ | |
-f deployStorage=false \ | |
-f deployStorageBroker=false \ | |
-f deployStorageController=false \ | |
-f branch=main \ | |
-f dockerTag=${{needs.tag.outputs.build-tag}} \ | |
-f deployPreprodRegion=true | |
gh workflow --repo neondatabase/aws run deploy-proxy-prod.yml --ref main \ | |
-f deployPgSniRouter=true \ | |
-f deployProxy=true \ | |
-f branch=main \ | |
-f dockerTag=${{needs.tag.outputs.build-tag}} | |
else | |
echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" | |
exit 1 | |
fi | |
- name: Create git tag | |
if: github.ref_name == 'release' || github.ref_name == 'release-proxy' | |
uses: actions/github-script@v7 | |
with: | |
# Retry script for 5XX server errors: https://github.com/actions/github-script#retries | |
retries: 5 | |
script: | | |
await github.rest.git.createRef({ | |
owner: context.repo.owner, | |
repo: context.repo.repo, | |
ref: "refs/tags/${{ needs.tag.outputs.build-tag }}", | |
sha: context.sha, | |
}) | |
# TODO: check how GitHub releases looks for proxy releases and enable it if it's ok | |
- name: Create GitHub release | |
if: github.ref_name == 'release' | |
uses: actions/github-script@v7 | |
with: | |
# Retry script for 5XX server errors: https://github.com/actions/github-script#retries | |
retries: 5 | |
script: | | |
await github.rest.repos.createRelease({ | |
owner: context.repo.owner, | |
repo: context.repo.repo, | |
tag_name: "${{ needs.tag.outputs.build-tag }}", | |
generate_release_notes: true, | |
}) | |
promote-compatibility-data: | |
needs: [ check-permissions, promote-images, tag, regress-tests ] | |
if: github.ref_name == 'release' | |
runs-on: [ self-hosted, gen3, small ] | |
container: | |
image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned | |
options: --init | |
steps: | |
- name: Promote compatibility snapshot for the release | |
env: | |
BUCKET: neon-github-public-dev | |
PREFIX: artifacts/latest | |
COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
run: | | |
# Update compatibility snapshot for the release | |
for pg_version in v14 v15 v16; do | |
for build_type in debug release; do | |
OLD_FILENAME=compatibility-snapshot-${build_type}-pg${pg_version}-${GITHUB_RUN_ID}.tar.zst | |
NEW_FILENAME=compatibility-snapshot-${build_type}-pg${pg_version}.tar.zst | |
time aws s3 mv --only-show-errors s3://${BUCKET}/${PREFIX}/${OLD_FILENAME} s3://${BUCKET}/${PREFIX}/${NEW_FILENAME} | |
done | |
done | |
# Update Neon artifact for the release (reuse already uploaded artifact) | |
for build_type in debug release; do | |
OLD_PREFIX=artifacts/${COMMIT_SHA}/${GITHUB_RUN_ID} | |
FILENAME=neon-${{ runner.os }}-${{ runner.arch }}-${build_type}-artifact.tar.zst | |
S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${OLD_PREFIX} | jq -r '.Contents[]?.Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true) | |
if [ -z "${S3_KEY}" ]; then | |
echo >&2 "Neither s3://${BUCKET}/${OLD_PREFIX}/${FILENAME} nor its version from previous attempts exist" | |
exit 1 | |
fi | |
time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/${PREFIX}/${FILENAME} | |
done | |
pin-build-tools-image: | |
needs: [ build-build-tools-image, promote-images, regress-tests ] | |
if: github.ref_name == 'main' | |
uses: ./.github/workflows/pin-build-tools-image.yml | |
with: | |
from-tag: ${{ needs.build-build-tools-image.outputs.image-tag }} | |
secrets: inherit | |
# This job simplifies setting branch protection rules (in GitHub UI) | |
# by allowing to set only this job instead of listing many others. | |
# It also makes it easier to rename or parametrise jobs (using matrix) | |
# which requires changes in branch protection rules | |
# | |
# Note, that we can't add external check (like `neon-cloud-e2e`) we still need to use GitHub UI for that. | |
# | |
# https://github.com/neondatabase/neon/settings/branch_protection_rules | |
conclusion: | |
if: always() | |
# Format `needs` differently to make the list more readable. | |
# Usually we do `needs: [...]` | |
needs: | |
- check-codestyle-python | |
- check-codestyle-rust | |
- regress-tests | |
- test-images | |
runs-on: ubuntu-22.04 | |
steps: | |
# The list of possible results: | |
# https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context | |
- name: Fail the job if any of the dependencies do not succeed | |
run: exit 1 | |
if: | | |
contains(needs.*.result, 'failure') | |
|| contains(needs.*.result, 'cancelled') | |
|| contains(needs.*.result, 'skipped') |