ci: fix the release pipeline

The release pipeline is currently editing the RocksDB version and in a
broken manner. Fix it and use a proper trigger for the artefacts release.

.github/workflows/artifact-release.yml

@@ -1,73 +1,112 @@
-name: Create release artifact
+name: Create release artifacts
 
 on:
-  workflow_dispatch:
+  push:
+    tags:
+      - 'speedb/v*'
+
+permissions:
+  contents: write # Needed for release assets upload
+  id-token: write # Needed for AWS credentials setting
 
 jobs:
+  build:
+    runs-on: [self-hosted, ubuntu, asrunner]
 
-  cleanup:
-    runs-on: [self-hosted, linux, x64, myrunner]
     container:
-      image: ubuntu:latest
-    steps:
-      - name: Cleaning up the $GITHUB_WORKSPACE as root from a Docker image
-        run: find /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/. -name . -o -prune -exec rm -rf -- {} + || true
-
-  build:
-    needs: cleanup
-    runs-on: [self-hosted, linux, x64, myrunner]
+      image: centos:7.9.2009
 
     steps:
-
+      - name: pre
+        run: |
+          yum install -y centos-release-scl epel-release
+          yum install -y make devtoolset-11-gcc-c++ \
+            coreutils wget unzip which git python3 openssl openssl-devel \
+            libzstd-devel lz4-devel snappy-devel zlib-devel \
+            java-1.8.0-openjdk-devel
+          echo "PATH=/opt/rh/devtoolset-11/root/usr/bin:${PATH}" >> $GITHUB_ENV
+          echo "RELEASE_VERSION=${GITHUB_REF_NAME#speedb/v}" >> $GITHUB_ENV
+
+      - name: Install CMake
+        run: |
+          CMAKE_RELEASE=3.20.1
+          wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_RELEASE}/cmake-${CMAKE_RELEASE}.tar.gz
+          tar xf cmake-${CMAKE_RELEASE}.tar.gz
+          cd cmake-${CMAKE_RELEASE}
+          ./bootstrap
+          make -j$(nproc) && make install
+          cd .. && rm -rf cmake-${CMAKE_RELEASE}*
+
+      - name: Install awscli
+        run: |
+          wget "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -O "awscliv2.zip"
+          unzip awscliv2.zip
+          ./aws/install
+          rm -rf aws awscliv2.zip
+
       - uses: actions/checkout@v3
 
-      - name: install dependencies
+      - run: mkdir "$GITHUB_WORKSPACE/out"
+
+      - name: Build and package release libraries
+        run: |
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DWITH_GFLAGS=0 -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1
+          mkdir -p "$GITHUB_WORKSPACE/out/root"
+          DESTDIR="$GITHUB_WORKSPACE/out/root" make -j$(nproc) install
+          ( cd "$GITHUB_WORKSPACE/out/root" && tar czf ../speedb-${RELEASE_VERSION}.tar.gz . )
+          rm -rf "$GITHUB_WORKSPACE/out/root"
+          cd .. && rm -rf build
+
+      - name: Build release Jar
         run: |
-          echo ~ && sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y docker-ce docker-ce-cli containerd.io unzip curl zip unzip
-          
-      - name: Configure AWS credentials from Test account
+          make clean
+          LIB_MODE=static DEBUG_LEVEL=0 PORTABLE=1 JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) rocksdbjavastatic
+          cp "java/target/speedbjni-${RELEASE_VERSION}-linux64.jar" "$GITHUB_WORKSPACE/out"
+
+      - name: Build db_bench
+        run: |
+          yum install -y gflags-devel
+          rm -rf build && mkdir build && cd build
+          cmake .. -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DWITH_GFLAGS=1 \
+            -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 \
+            -DWITH_BENCHMARK_TOOLS=1 -DROCKSDB_BUILD_SHARED=1
+          make -j$(nproc) db_bench
+          tar czf "$GITHUB_WORKSPACE/out/db_bench-speedb-${RELEASE_VERSION}.tar.gz" db_bench
+          cd .. && rm -rf build
+
+      - name: Generate checksums
+        run: |
+          for f in $GITHUB_WORKSPACE/out/*; do
+            sha256sum "$f" > "$f.sha256"
+          done
+
+      - name: Get release date
+        run: |
+          echo "RELEASE_DATE=$(git for-each-ref "--format=%(creatordate:short)" "refs/tags/${GITHUB_REF_NAME}")" >> $GITHUB_ENV
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          generate_release_notes: false
+          name: Speedb ${{ env.RELEASE_VERSION }} (${{ env.RELEASE_DATE }})
+          files: |
+            out/db_bench-speedb-${{ env.RELEASE_VERSION }}.tar.gz
+            out/db_bench-speedb-${{ env.RELEASE_VERSION }}.tar.gz.sha256
+            out/speedb-${{ env.RELEASE_VERSION }}.tar.gz
+            out/speedb-${{ env.RELEASE_VERSION }}.tar.gz.sha256
+            out/speedbjni-${{ env.RELEASE_VERSION }}-linux64.jar
+            out/speedbjni-${{ env.RELEASE_VERSION }}-linux64.jar.sha256
+
+      - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-2
 
-      - name: docker run
-        run: |
-          export BUILD_OUTPUT_DIR=~/build
-          echo $BUILD_OUTPUT_DIR
-          rm -rf $BUILD_OUTPUT_DIR
-          mkdir -p $BUILD_OUTPUT_DIR
-          docker build -f ./build_tools/Dockerfile -t spdb-build .
-          docker run --rm -v $(readlink -f ${BUILD_OUTPUT_DIR}):/out -v $GITHUB_WORKSPACE:/speedb -e GITHUB_WORKSPACE=$GITHUB_WORKSPACE spdb-build
-          ls -alFh $BUILD_OUTPUT_DIR
-          zip --junk-paths "Release-$GITHUB_REF_NAME.zip" $BUILD_OUTPUT_DIR/*
-
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: Release ${{ github.ref_name }}
-          draft: false
-          prerelease: false
-      - name: Upload Release Asset
-        id: upload-release-asset 
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 
-          asset_path: ./Release-${{ github.ref_name}}.zip
-          asset_name: Release-${{ github.ref_name}}.zip
-          asset_content_type: application/zip
-
       - name: Upload artifacts to S3
         run: |
-          export BUILD_OUTPUT_DIR=~/build
-          echo $BUILD_OUTPUT_DIR
-          aws s3 cp $BUILD_OUTPUT_DIR s3://spdb-github-artifacts/"release-$GITHUB_REF_NAME" --recursive
-          aws s3 ls s3://spdb-github-artifacts/
-          rm -rf $BUILD_OUTPUT_DIR
+          aws s3 cp "$GITHUB_WORKSPACE/out" "s3://spdb-github-artifacts/release-${RELEASE_VERSION}" --recursive
+          rm -rf "$GITHUB_WORKSPACE/out"

.github/workflows/new_release_line.yml

@@ -5,79 +5,102 @@ on:
     inputs:
       new_branch_major:
         description: "Next release Major version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+        required: false
       new_branch_minor:
         description: "Next release Minor version (LEAVE EMPTY FOR AUTO-INCREMENT)"
+        required: false
+    branches:
+      - main
+      - 'release/*'
 
+permissions:
+  contents: read
 
 jobs:
-  build:
+  tag_version:
     runs-on: [self-hosted, ubuntu, asrunner]
     container:
       image: alpine:3.14
 
+    env:
+      VERSION_FILE: speedb/version.h
+
     steps:
-      - name: pre-run
+      - name: pre
         run: |
           echo "nameserver 8.8.8.8" > /etc/resolv.conf
-          apk update && apk add git
+          apk add git openssh-client
+
+      - name: Verify chosen version
+        run: |
+          if ! echo "${{ inputs.new_branch_major }}" | grep -q "^[0-9]*$"; then
+            echo >&2 "error: major version must be a positive number" && false
+          fi
+          if ! echo "${{ inputs.new_branch_minor }}" | grep -q "^[0-9]*$"; then
+            echo >&2 "error: minor version must be a positive number" && false
+          fi
+
+          if [ "${{ inputs.new_branch_major }}${{ inputs.new_branch_minor }}" != "" ] && [ "$GITHUB_REF" != "refs/heads/main" ]; then
+            echo >&2 "error: cannot cut a major or a minor release from a branch that isn't main" && false
+          elif [ "$GITHUB_REF" != "refs/heads/main" ] && ! echo "$GITHUB_REF" | grep -q "^refs/heads/release/"; then
+            echo "error: cannot cut a patch release from a non-release branch" && false
+          fi
           
       - uses: actions/checkout@v3
+        with:
+          ssh-key: ${{ secrets.RELEASE_SSH_KEY }}
 
-      - name: Calculate tag
-        run: |
-          git config --global --add safe.directory /__w/rocksdb/rocksdb
-          git config user.name "GitHub Actions Bot"
-          git config user.email "<>"
-          git pull
-          major=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MAJOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
-          minor=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MINOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
-          patch=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_PATCH' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1)
-          new_minor=$(echo $minor + 1 | bc)
-          echo $minor
-          echo "major=$major" >> $GITHUB_ENV
-          echo "minor=$minor" >> $GITHUB_ENV
-          echo "new_minor=$new_minor" >> $GITHUB_ENV
-          current_version="$major.$minor.99"
-          echo $current_version
-      
-      - name: Create release branch
+      - name: Calculate new version
         run: |
-          if [ -z "${{ inputs.new_branch_major }}" ]; then
-            echo "empty"
-            new_branch="release/${{ env.major }}.${{ env.minor }}"
-            echo $new_branch
-            git checkout -b $new_branch
-            git push -u origin $new_branch
+          major=$(grep '_MAJOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//')
+          minor=$(grep '_MINOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//')
+
+          if [ -n "${{ inputs.new_branch_major }}" ] && [ "${{ inputs.new_branch_major }}" -lt "$major" ]; then
+            echo >&2 "error: the chosen major version is lower than current one" && false
+          elif [ -n "${{ inputs.new_branch_major }}" ] && [ "${{ inputs.new_branch_major }}" -gt "$major" ]; then
+            major=${{ inputs.new_branch_major }}
+            if [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -ne 0 ]; then
+              echo >&2 "error: cannot bump minor version when bumping major version" && false
+            fi
+            minor=0
+            patch=0
+          elif [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -lt "$minor" ]; then
+            echo >&2 "error: the chosen minor version is lower than current one" && false
+          elif [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -gt "$minor" ]; then
+            minor=${{ inputs.new_branch_minor }}
+            patch=0
+          elif [ "$GITHUB_REF" = "refs/heads/main" ]; then
+            minor=$(( $minor + 1 ))
+            patch=0
           else
-            echo "minor=$new_minor" >> $GITHUB_ENV
-            echo "not empty"
-            new_branch="release/${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}"
-            echo $new_branch
-            git checkout -b $new_branch
-            git push -u origin $new_branch
+            patch=$(( $(grep '_PATCH\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') + 1 ))
           fi
+
+          echo "major=$major" >> $GITHUB_ENV
+          echo "minor=$minor" >> $GITHUB_ENV
+          echo "patch=$patch" >> $GITHUB_ENV
+
       - name: Update version.h
         run: |
-          if [ -z "${{ inputs.new_branch_major }}" ]; then
-            git checkout main
-            git tag "v${{ env.major }}.${{ env.minor }}"
-            git push --tags
-            sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ env.major }}/' include/rocksdb/version.h
-            sed -i -e 's/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR ${{ env.new_minor }}/' include/rocksdb/version.h
-            sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h
-            cat include/rocksdb/version.h
-            git commit -am "updated version.h version"
-            git push
-          else
-            git checkout main
-            new_minor=$(echo ${{ inputs.new_branch_minor }} + 1 | bc)
-            echo $new_minor
-            git tag "v${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}"
-            git push --tags
-            sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ inputs.new_branch_major }}/' include/rocksdb/version.h
-            sed -i -e "s/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR $new_minor/g" include/rocksdb/version.h
-            sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h
-            cat include/rocksdb/version.h
-            git commit -am "updated version.h version"
-            git push
+          git config user.name "GitHub Runner Bot"
+          git config user.email "<>"
+
+          sed -i -e "s/\(#define [^\s]\+_MAJOR\s\+\)[0-9]\+/\1${major}/" "$VERSION_FILE"
+          sed -i -e "s/\(#define [^\s]\+_MINOR\s\+\)[0-9]\+/\1${minor}/" "$VERSION_FILE"
+          sed -i -e "s/\(#define [^\s]\+_PATCH\s\+\)[0-9]\+/\1${patch}/" "$VERSION_FILE"
+
+          git add "$VERSION_FILE"
+          git commit -m "release: publish version ${major}.${minor}.${patch}"
+          git push origin ${GITHUB_REF#refs/heads/}
+
+      - name: Tag and release
+        run: |
+          # Create a branch if it's a major or a minor release
+          if [ "$patch" -eq 0 ]; then
+            git checkout -b "release/${major}.${minor}"
+            git push -u origin "release/${major}.${minor}"
           fi
+
+          # Create a tag for the release
+          git tag "speedb/v${major}.${minor}.${patch}"
+          git push origin "speedb/v${major}.${minor}.${patch}"