diff --git a/.github/workflows/artifact-release.yml b/.github/workflows/artifact-release.yml index e0a3a1b91b..2768779e75 100644 --- a/.github/workflows/artifact-release.yml +++ b/.github/workflows/artifact-release.yml @@ -1,73 +1,112 @@ -name: Create release artifact +name: Create release artifacts on: - workflow_dispatch: + push: + tags: + - 'speedb/v*' + +permissions: + contents: write # Needed for release assets upload + id-token: write # Needed for AWS credentials setting jobs: + build: + runs-on: [self-hosted, ubuntu, asrunner] - cleanup: - runs-on: [self-hosted, linux, x64, myrunner] container: - image: ubuntu:latest - steps: - - name: Cleaning up the $GITHUB_WORKSPACE as root from a Docker image - run: find /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/. -name . -o -prune -exec rm -rf -- {} + || true - - build: - needs: cleanup - runs-on: [self-hosted, linux, x64, myrunner] + image: centos:7.9.2009 steps: - + - name: pre + run: | + yum install -y centos-release-scl epel-release + yum install -y make devtoolset-11-gcc-c++ \ + coreutils wget unzip which git python3 openssl openssl-devel \ + libzstd-devel lz4-devel snappy-devel zlib-devel \ + java-1.8.0-openjdk-devel + echo "PATH=/opt/rh/devtoolset-11/root/usr/bin:${PATH}" >> $GITHUB_ENV + echo "RELEASE_VERSION=${GITHUB_REF_NAME#speedb/v}" >> $GITHUB_ENV + + - name: Install CMake + run: | + CMAKE_RELEASE=3.20.1 + wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_RELEASE}/cmake-${CMAKE_RELEASE}.tar.gz + tar xf cmake-${CMAKE_RELEASE}.tar.gz + cd cmake-${CMAKE_RELEASE} + ./bootstrap + make -j$(nproc) && make install + cd .. && rm -rf cmake-${CMAKE_RELEASE}* + + - name: Install awscli + run: | + wget "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -O "awscliv2.zip" + unzip awscliv2.zip + ./aws/install + rm -rf aws awscliv2.zip + - uses: actions/checkout@v3 - - name: install dependencies + - run: mkdir "$GITHUB_WORKSPACE/out" + + - name: Build and package release libraries + run: | + rm -rf build && mkdir build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DWITH_GFLAGS=0 -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 + mkdir -p "$GITHUB_WORKSPACE/out/root" + DESTDIR="$GITHUB_WORKSPACE/out/root" make -j$(nproc) install + ( cd "$GITHUB_WORKSPACE/out/root" && tar czf ../speedb-${RELEASE_VERSION}.tar.gz . ) + rm -rf "$GITHUB_WORKSPACE/out/root" + cd .. && rm -rf build + + - name: Build release Jar run: | - echo ~ && sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y docker-ce docker-ce-cli containerd.io unzip curl zip unzip - - - name: Configure AWS credentials from Test account + make clean + LIB_MODE=static DEBUG_LEVEL=0 PORTABLE=1 JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) rocksdbjavastatic + cp "java/target/speedbjni-${RELEASE_VERSION}-linux64.jar" "$GITHUB_WORKSPACE/out" + + - name: Build db_bench + run: | + yum install -y gflags-devel + rm -rf build && mkdir build && cd build + cmake .. -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DWITH_GFLAGS=1 \ + -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1 \ + -DWITH_BENCHMARK_TOOLS=1 -DROCKSDB_BUILD_SHARED=1 + make -j$(nproc) db_bench + tar czf "$GITHUB_WORKSPACE/out/db_bench-speedb-${RELEASE_VERSION}.tar.gz" db_bench + cd .. && rm -rf build + + - name: Generate checksums + run: | + for f in $GITHUB_WORKSPACE/out/*; do + sha256sum "$f" > "$f.sha256" + done + + - name: Get release date + run: | + echo "RELEASE_DATE=$(git for-each-ref "--format=%(creatordate:short)" "refs/tags/${GITHUB_REF_NAME}")" >> $GITHUB_ENV + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + generate_release_notes: false + name: Speedb ${{ env.RELEASE_VERSION }} (${{ env.RELEASE_DATE }}) + files: | + out/db_bench-speedb-${{ env.RELEASE_VERSION }}.tar.gz + out/db_bench-speedb-${{ env.RELEASE_VERSION }}.tar.gz.sha256 + out/speedb-${{ env.RELEASE_VERSION }}.tar.gz + out/speedb-${{ env.RELEASE_VERSION }}.tar.gz.sha256 + out/speedbjni-${{ env.RELEASE_VERSION }}-linux64.jar + out/speedbjni-${{ env.RELEASE_VERSION }}-linux64.jar.sha256 + + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: eu-west-2 - - name: docker run - run: | - export BUILD_OUTPUT_DIR=~/build - echo $BUILD_OUTPUT_DIR - rm -rf $BUILD_OUTPUT_DIR - mkdir -p $BUILD_OUTPUT_DIR - docker build -f ./build_tools/Dockerfile -t spdb-build . - docker run --rm -v $(readlink -f ${BUILD_OUTPUT_DIR}):/out -v $GITHUB_WORKSPACE:/speedb -e GITHUB_WORKSPACE=$GITHUB_WORKSPACE spdb-build - ls -alFh $BUILD_OUTPUT_DIR - zip --junk-paths "Release-$GITHUB_REF_NAME.zip" $BUILD_OUTPUT_DIR/* - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: Release ${{ github.ref_name }} - draft: false - prerelease: false - - name: Upload Release Asset - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: ./Release-${{ github.ref_name}}.zip - asset_name: Release-${{ github.ref_name}}.zip - asset_content_type: application/zip - - name: Upload artifacts to S3 run: | - export BUILD_OUTPUT_DIR=~/build - echo $BUILD_OUTPUT_DIR - aws s3 cp $BUILD_OUTPUT_DIR s3://spdb-github-artifacts/"release-$GITHUB_REF_NAME" --recursive - aws s3 ls s3://spdb-github-artifacts/ - rm -rf $BUILD_OUTPUT_DIR + aws s3 cp "$GITHUB_WORKSPACE/out" "s3://spdb-github-artifacts/release-${RELEASE_VERSION}" --recursive + rm -rf "$GITHUB_WORKSPACE/out" diff --git a/.github/workflows/new_release_line.yml b/.github/workflows/new_release_line.yml index 2fa674cfea..b835b7a9c7 100644 --- a/.github/workflows/new_release_line.yml +++ b/.github/workflows/new_release_line.yml @@ -5,79 +5,102 @@ on: inputs: new_branch_major: description: "Next release Major version (LEAVE EMPTY FOR AUTO-INCREMENT)" + required: false new_branch_minor: description: "Next release Minor version (LEAVE EMPTY FOR AUTO-INCREMENT)" + required: false + branches: + - main + - 'release/*' +permissions: + contents: read jobs: - build: + tag_version: runs-on: [self-hosted, ubuntu, asrunner] container: image: alpine:3.14 + env: + VERSION_FILE: speedb/version.h + steps: - - name: pre-run + - name: pre run: | echo "nameserver 8.8.8.8" > /etc/resolv.conf - apk update && apk add git + apk add git openssh-client + + - name: Verify chosen version + run: | + if ! echo "${{ inputs.new_branch_major }}" | grep -q "^[0-9]*$"; then + echo >&2 "error: major version must be a positive number" && false + fi + if ! echo "${{ inputs.new_branch_minor }}" | grep -q "^[0-9]*$"; then + echo >&2 "error: minor version must be a positive number" && false + fi + + if [ "${{ inputs.new_branch_major }}${{ inputs.new_branch_minor }}" != "" ] && [ "$GITHUB_REF" != "refs/heads/main" ]; then + echo >&2 "error: cannot cut a major or a minor release from a branch that isn't main" && false + elif [ "$GITHUB_REF" != "refs/heads/main" ] && ! echo "$GITHUB_REF" | grep -q "^refs/heads/release/"; then + echo "error: cannot cut a patch release from a non-release branch" && false + fi - uses: actions/checkout@v3 + with: + ssh-key: ${{ secrets.RELEASE_SSH_KEY }} - - name: Calculate tag - run: | - git config --global --add safe.directory /__w/rocksdb/rocksdb - git config user.name "GitHub Actions Bot" - git config user.email "<>" - git pull - major=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MAJOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1) - minor=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MINOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1) - patch=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_PATCH' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1) - new_minor=$(echo $minor + 1 | bc) - echo $minor - echo "major=$major" >> $GITHUB_ENV - echo "minor=$minor" >> $GITHUB_ENV - echo "new_minor=$new_minor" >> $GITHUB_ENV - current_version="$major.$minor.99" - echo $current_version - - - name: Create release branch + - name: Calculate new version run: | - if [ -z "${{ inputs.new_branch_major }}" ]; then - echo "empty" - new_branch="release/${{ env.major }}.${{ env.minor }}" - echo $new_branch - git checkout -b $new_branch - git push -u origin $new_branch + major=$(grep '_MAJOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') + minor=$(grep '_MINOR\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') + + if [ -n "${{ inputs.new_branch_major }}" ] && [ "${{ inputs.new_branch_major }}" -lt "$major" ]; then + echo >&2 "error: the chosen major version is lower than current one" && false + elif [ -n "${{ inputs.new_branch_major }}" ] && [ "${{ inputs.new_branch_major }}" -gt "$major" ]; then + major=${{ inputs.new_branch_major }} + if [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -ne 0 ]; then + echo >&2 "error: cannot bump minor version when bumping major version" && false + fi + minor=0 + patch=0 + elif [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -lt "$minor" ]; then + echo >&2 "error: the chosen minor version is lower than current one" && false + elif [ -n "${{ inputs.new_branch_minor }}" ] && [ "${{ inputs.new_branch_minor }}" -gt "$minor" ]; then + minor=${{ inputs.new_branch_minor }} + patch=0 + elif [ "$GITHUB_REF" = "refs/heads/main" ]; then + minor=$(( $minor + 1 )) + patch=0 else - echo "minor=$new_minor" >> $GITHUB_ENV - echo "not empty" - new_branch="release/${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}" - echo $new_branch - git checkout -b $new_branch - git push -u origin $new_branch + patch=$(( $(grep '_PATCH\s\+[0-9]\+' "$VERSION_FILE" | sed 's/[^0-9]\+//') + 1 )) fi + + echo "major=$major" >> $GITHUB_ENV + echo "minor=$minor" >> $GITHUB_ENV + echo "patch=$patch" >> $GITHUB_ENV + - name: Update version.h run: | - if [ -z "${{ inputs.new_branch_major }}" ]; then - git checkout main - git tag "v${{ env.major }}.${{ env.minor }}" - git push --tags - sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ env.major }}/' include/rocksdb/version.h - sed -i -e 's/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR ${{ env.new_minor }}/' include/rocksdb/version.h - sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h - cat include/rocksdb/version.h - git commit -am "updated version.h version" - git push - else - git checkout main - new_minor=$(echo ${{ inputs.new_branch_minor }} + 1 | bc) - echo $new_minor - git tag "v${{ inputs.new_branch_major }}.${{ inputs.new_branch_minor }}" - git push --tags - sed -i -e 's/#define ROCKSDB_MAJOR.*/#define ROCKSDB_MAJOR ${{ inputs.new_branch_major }}/' include/rocksdb/version.h - sed -i -e "s/#define ROCKSDB_MINOR.*/#define ROCKSDB_MINOR $new_minor/g" include/rocksdb/version.h - sed -i -e 's/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH 99/' include/rocksdb/version.h - cat include/rocksdb/version.h - git commit -am "updated version.h version" - git push + git config user.name "GitHub Runner Bot" + git config user.email "<>" + + sed -i -e "s/\(#define [^\s]\+_MAJOR\s\+\)[0-9]\+/\1${major}/" "$VERSION_FILE" + sed -i -e "s/\(#define [^\s]\+_MINOR\s\+\)[0-9]\+/\1${minor}/" "$VERSION_FILE" + sed -i -e "s/\(#define [^\s]\+_PATCH\s\+\)[0-9]\+/\1${patch}/" "$VERSION_FILE" + + git add "$VERSION_FILE" + git commit -m "release: publish version ${major}.${minor}.${patch}" + git push origin ${GITHUB_REF#refs/heads/} + + - name: Tag and release + run: | + # Create a branch if it's a major or a minor release + if [ "$patch" -eq 0 ]; then + git checkout -b "release/${major}.${minor}" + git push -u origin "release/${major}.${minor}" fi + + # Create a tag for the release + git tag "speedb/v${major}.${minor}.${patch}" + git push origin "speedb/v${major}.${minor}.${patch}" diff --git a/.github/workflows/versioning.yml b/.github/workflows/versioning.yml deleted file mode 100644 index 45c5d73718..0000000000 --- a/.github/workflows/versioning.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: versioning - -on: - #push: - workflow_dispatch: - workflow_call: - -jobs: - UpdateFullVer: - runs-on: [self-hosted, ubuntu, asrunner] - container: - image: ubuntu:latest - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Update version - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE - git config user.name "GitHub Actions Bot" - git config user.email "<>" - git pull - git fetch --tags - echo $GITHUB_HEAD_REF - echo $GITHUB_REF_NAME - major=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MAJOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1) - minor=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_MINOR' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1) - patch=$(cat $GITHUB_WORKSPACE/include/rocksdb/version.h | grep '#define ROCKSDB_PATCH' | grep -o '[^,]\+$' | tr ' ' '\n' | tail -n1) - if [ $GITHUB_REF_NAME == "main" ]; then - new_patch=$(echo $patch + 1 | bc) - sed -i -e "s/#define ROCKSDB_PATCH.*/#define ROCKSDB_PATCH $new_patch/g" include/rocksdb/version.h - sed -i -e "s/#define ROCKSDB_BRANCH.*/#define ROCKSDB_BRANCH $GITHUB_REF_NAME/g" include/rocksdb/version.h - git commit -am "updated version.h version" - git push - git tag "v$major.$minor.$new_patch-$GITHUB_REF_NAME" - git push --tags - elif echo $GITHUB_REF_NAME | grep -Eq 'release/'; then - echo "##### FEATURE BRANCH, SKIPPING #####" - else - if ( git tag -l --sort=-version:refname | grep "v$major.$minor.$patch" | grep -E -i "v([0-9]+).([0-9]+).([0-9]+).([0-9]+)" ); then - current_feature=$(git tag -l --sort=-version:refname | grep "v$major.$minor.$patch" | grep -E -i "v([0-9]+).([0-9]+).([0-9]+).([0-9]+)" | head -1 | tr '.' '\n' | tail -n1) - new_patch=$(echo $patch + 1 | bc) - new_feature=$(echo $current_feature + 1 | bc) - git tag "v$major.$minor.$patch.$new_feature" - git push --tags - else - git tag "v$major.$minor.$patch.0" - git push --tags - fi - fi diff --git a/build_tools/Dockerfile b/build_tools/Dockerfile deleted file mode 100644 index 5ddcc0e93a..0000000000 --- a/build_tools/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM docker.io/centos:7.9.2009 - -# Install required libraries and utilities for building Speedb -RUN yum install -y centos-release-scl epel-release -RUN yum install -y make devtoolset-9-gcc-c++ \ - coreutils wget which git python3 openssl-devel \ - libzstd-devel lz4-devel snappy-devel zlib-devel \ - java-1.8.0-openjdk-devel -ENV PATH="/opt/rh/devtoolset-9/root/usr/bin:${PATH}" - -# Install CMake -RUN wget https://github.com/Kitware/CMake/releases/download/v3.20.1/cmake-3.20.1.tar.gz -RUN tar xf cmake-3.20.1.tar.gz -RUN cd cmake-3.20.1 && ./bootstrap && make -j $(nproc) && make install && cd .. && rm -rf cmake-3.20.1* - -RUN echo -e "#!/bin/sh\n\ -set -xeo pipefail\n\ -java -version\n\ -cd speedb\n\ -rm -rf build && mkdir build && cd build\n\ -cmake .. -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DWITH_GFLAGS=0 -DWITH_SNAPPY=1 -DWITH_LZ4=1 -DWITH_ZLIB=1 -DWITH_ZSTD=1\n\ -make -j$(nproc) speedb speedb-shared\n\ -cp libspeedb.{a,so} /out\n\ -cd .. && rm -rf build\n\ -make clean && LIB_MODE=static DEBUG_LEVEL=0 PORTABLE=1 JAVA_HOME=/usr/lib/jvm/java-openjdk make -j$(nproc) rocksdbjavastatic\n\ -cp java/target/speedbjni-*.jar /out" > /usr/bin/build-speedb.sh - -RUN chmod +x /usr/bin/build-speedb.sh -ENTRYPOINT ["/usr/bin/build-speedb.sh"] - -# Declare volumes -VOLUME ["/speedb", "/out"]