inital commit

.github/workflows/build-test-release.yml

@@ -0,0 +1,251 @@
+name: CI/CD
+
+on:
+  push:
+    branches:
+    - main
+    - '*\@[0-9]+.[0-9]+.[0-9]+*'  # package development branch
+
+jobs:
+  build:
+    if: ${{ !contains(github.event.head_commit.message, '[no ci]') }}
+    runs-on: ubuntu-latest
+    outputs:
+      branch: ${{ steps.get_pkg_info.outputs.branch }}
+      pkg_name: ${{ steps.get_pkg_info.outputs.pkg_name }}
+      pkg_ver: ${{ steps.get_pkg_info.outputs.pkg_ver }}
+      docker_file: ${{ steps.get_pkg_info.outputs.docker_file }}
+      repo_lowercase: ${{ steps.repo_lowercase.outputs.lowercase }}
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+
+    - name: Extract package name and version from branch name
+      id: get_pkg_info
+      shell: bash
+      run: |
+        echo "::set-output name=branch::$(echo ${GITHUB_REF#refs/heads/})"
+        PKG_NAME=$(echo ${GITHUB_REF#refs/heads/} | awk -F'@' '{print $1}')
+        PKG_VER=$(echo ${GITHUB_REF#refs/heads/} | awk -F'@' '{print $2}')
+        echo "::set-output name=pkg_name::$(echo $PKG_NAME)"
+        echo "::set-output name=pkg_ver::$(echo $PKG_VER)"
+        if [[ -f "./$PKG_NAME/Dockerfile" ]]; then
+          echo "::set-output name=docker_file::$(echo ./$PKG_NAME/Dockerfile)"
+        else
+          echo "::set-output name=docker_file::"
+        fi
+
+    - name: Login to GitHub Container Registry
+      if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}}
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.CR_PAT }}
+
+    - id: repo_lowercase
+      uses: ASzc/change-string-case-action@v1
+      with:
+        string: ${{ github.repository }}
+
+    - name: Build image
+      if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}}
+      uses: docker/build-push-action@v2
+      with:
+        context: "./${{steps.get_pkg_info.outputs.pkg_name}}"
+        file: "${{steps.get_pkg_info.outputs.docker_file}}"
+        platforms: linux/amd64
+        load: true
+        tags: |
+          ghcr.io/${{ steps.repo_lowercase.outputs.lowercase }}.${{ steps.get_pkg_info.outputs.pkg_name }}:${{ steps.get_pkg_info.outputs.pkg_ver }}
+
+    - name: Push image to ghcr.io
+      id: push_to_ghcr
+      if: ${{ steps.get_pkg_info.outputs.branch != 'main' && steps.get_pkg_info.outputs.docker_file != ''}}
+      shell: bash
+      run: |
+        docker push ghcr.io/${{ steps.repo_lowercase.outputs.lowercase }}.${{ steps.get_pkg_info.outputs.pkg_name }}:${{ steps.get_pkg_info.outputs.pkg_ver }} | tee ./ghcr.io.stdout
+        IMAGE_SHA=$(cat ./ghcr.io.stdout | tr ' ' '\n' | grep 'sha256:' | awk -F':' '{print $2}')
+        echo "::set-output name=image_sha256::$(echo ${IMAGE_SHA})"
+        echo "::set-output name=created_at::$(date -u +%FT%TZ)"
+
+
+  test-and-release:
+    runs-on: ubuntu-latest
+    needs: [build]
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install wfpm
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
+        if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+    - name: Install Nextflow
+      run: |
+        wget --tries=10 -qO- https://get.nextflow.io | bash
+        sudo chmod 755 nextflow
+        sudo mv nextflow /usr/local/bin/
+
+    - name: "Login to GitHub Container Registry"  # normally shouldn't need to loging, but new image when just created is private
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.CR_PAT }}
+
+    - name: Run tests for all packages
+      if: ${{ needs.build.outputs.branch == 'main' }}
+      run: |
+        wfpm test
+
+    - name: Run tests for the current package only
+      if: ${{ needs.build.outputs.branch != 'main' }}
+      run: |
+        cd ./${{needs.build.outputs.pkg_name}}
+        wfpm test
+
+    - name: Check whether to proceed with release
+      id: to_release
+      run: |
+        if [[ \
+          "${{ github.event.head_commit.message }}" = "Merge"[[:space:]]"pull"[[:space:]]"request"* && \
+          "${{ github.event.head_commit.message }}" = *"@"* && \
+          "${{ github.event.head_commit.message }}" = *"[release]"* && \
+          "${{ needs.build.outputs.branch }}" = 'main'
+        ]]; then
+          echo "::set-output name=release::$(echo Y)"
+        else
+          echo "::set-output name=release::$(echo N)"
+        fi
+
+    - name: Extract package name and version from commit message
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      id: get_pkg_info
+      shell: bash
+      run: |
+        MERGED_BR=$(echo -e '${{ github.event.head_commit.message }}' | \
+          { grep '^Merge pull request'||:; } | tr ' ' '\n' |{ grep '@'||:; } | awk -F'/' '{print $2}')
+        PKG_NAME=$(echo $MERGED_BR | awk -F'@' '{print $1}')
+        PKG_VER=$(echo $MERGED_BR | awk -F'@' '{print $2}')
+        echo "::set-output name=pkg_name::$(echo ${PKG_NAME})"
+        echo "::set-output name=pkg_ver::$(echo ${PKG_VER})"
+        if [[ -f "./$PKG_NAME/Dockerfile" ]]; then
+          echo "::set-output name=docker_file::$(echo ./$PKG_NAME/Dockerfile)"
+          echo "::set-output name=docker_image::$(echo ghcr.io/${{ needs.build.outputs.repo_lowercase }}.$PKG_NAME:$PKG_VER)"
+        else
+          echo "::set-output name=docker_file::"
+          echo "::set-output name=docker_image::"
+        fi
+
+    - name: Prepare package release tarball
+      id: prep_assets
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      shell: bash
+      run: |
+        ./scripts/cleanup_temp_files.sh  # just in case
+        PKG_TAR=${{ steps.get_pkg_info.outputs.pkg_name }}.${{ steps.get_pkg_info.outputs.pkg_ver }}.tar.gz
+        pushd ${{ steps.get_pkg_info.outputs.pkg_name }}
+        tar --exclude=wfpr_modules --dereference -czvf ../$PKG_TAR .
+        popd
+        echo "::set-output name=pkg_tar::$(echo ${PKG_TAR})"
+        echo "::set-output name=pkg_tar_sha::$(sha256sum ${PKG_TAR} |awk '{print $1}')"
+        echo "::set-output name=pkg_tar_size::$(ls -l ${PKG_TAR} |awk '{print $5}')"
+        echo "::set-output name=created_at::$(date -u +%FT%TZ)"
+
+    - name: Create release
+      id: create_release
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        tag_name: ${{ steps.get_pkg_info.outputs.pkg_name }}.${{ steps.get_pkg_info.outputs.pkg_ver }}
+        release_name: ${{ steps.get_pkg_info.outputs.pkg_name }}.${{ steps.get_pkg_info.outputs.pkg_ver }}
+        body: |
+          * Release `${{ steps.get_pkg_info.outputs.pkg_name }}.${{ steps.get_pkg_info.outputs.pkg_ver }}` (${{ github.sha }})
+          * Package `${{ steps.prep_assets.outputs.pkg_tar }}` (sha256: `${{ steps.prep_assets.outputs.pkg_tar_sha }}`)
+        draft: false
+        prerelease: false
+
+    - name: Prepare in pkg-release.json
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      shell: bash
+      run: |
+        if [[ '${{ steps.get_pkg_info.outputs.docker_file }}' != '' ]]; then
+          docker pull ${{ steps.get_pkg_info.outputs.docker_image }} | tee image.info
+          IMAGE_SHA=$(cat ./image.info | tr ' ' '\n' | grep 'sha256:' | awk -F':' '{print $2}')
+        else
+          IMAGE_SHA=$(echo)
+        fi
+
+        # temporary solution
+        TAG="${{ steps.get_pkg_info.outputs.pkg_name }}.${{ steps.get_pkg_info.outputs.pkg_ver }}"
+        ./scripts/prepare_package_release_json.py -p ${{ steps.get_pkg_info.outputs.pkg_name }}/pkg.json -d \
+          "
+          {
+            \"_release\": {
+              \"created\": \"${{ steps.prep_assets.outputs.created_at }}\",
+              \"hash\": {
+                \"checksum_type\": \"sha1\",
+                \"checksume\": \"${{ github.sha }}\"
+              },
+              \"tag\": \"$TAG\",
+              \"assets\": [
+                {
+                  \"filename\": \"$TAG.tar.gz\",
+                  \"download_url\": \"https://github.com/${{ github.repository }}/releases/download/$TAG/$TAG.tar.gz\",
+                  \"checksum\": \"${{ steps.prep_assets.outputs.pkg_tar_sha }}\",
+                  \"checksum_type\": \"sha256\",
+                  \"size\": ${{ steps.prep_assets.outputs.pkg_tar_size }}
+                },
+                {
+                  \"_NOTE_\": \"this file\",
+                  \"filename\": \"pkg-release.json\",
+                  \"download_url\": \"https://github.com/${{ github.repository }}/releases/download/$TAG/pkg-release.json\",
+                  \"checksum\": null,
+                  \"checksum_type\": null,
+                  \"size\": null
+                }
+              ]
+            },
+            \"_image_digest\": {
+              \"checksum\": \"$IMAGE_SHA\",
+              \"checksum_type\": \"sha256\"
+            }
+          }
+          " > pkg-release.json
+
+    - name: Upload package release tarball
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./${{ steps.prep_assets.outputs.pkg_tar }}
+        asset_name: ${{ steps.prep_assets.outputs.pkg_tar }}
+        asset_content_type: application/zip
+
+    - name: Upload package release json
+      if: ${{ steps.to_release.outputs.release == 'Y' }}
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ steps.create_release.outputs.upload_url }}
+        asset_path: ./pkg-release.json
+        asset_name: pkg-release.json
+        asset_content_type: application/json

.gitignore

@@ -0,0 +1,70 @@
+*.py[cod]
+
+# C extensions
+*.so
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+.eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+venv*/
+pyvenv*/
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+.coverage
+.tox
+.coverage.*
+nosetests.xml
+coverage.xml
+htmlcov
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+.idea
+*.iml
+*.komodoproject
+
+# Complexity
+output/*.html
+output/*/index.html
+
+# Sphinx
+docs/_build
+
+.DS_Store
+*~
+.*.sw[po]
+.build
+.ve
+.env
+.cache
+.pytest
+.bootstrap
+.appveyor.token
+*.bak
+*.log
+.vscode
+.python-version
+.nextflow*
+work
+outdir
+wfpr_modules/github.com/*/*/*/tests

.wfpm

@@ -0,0 +1,5 @@
+project_name: demo-tool-pkgs
+license: MIT
+repo_type: git
+repo_server: github.com
+repo_account: icgc-tcga-pancancer

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2021, PCAWG
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

README.md

@@ -0,0 +1,3 @@
+# Demo Tool Packages
+
+Update this to describe your awesome project.

nextflow.config

@@ -0,0 +1,10 @@
+manifest {
+    homePage = 'https://github.com/icgc-tcga-pancancer/demo-tool-pkgs'
+    description = 'Demo Tool Packages'
+    nextflowVersion = '>=20.10'
+}
+
+docker {
+    enabled = true
+    runOptions = '-u \$(id -u):\$(id -g)'
+}

scripts/cleanup_temp_files.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+for f in `find . -type d |egrep '(/work$)|(/.nextflow$)|(/outdir$)|(.nextflow.log)'`; do
+    echo remove $f;
+    rm -fr $f;
+done
+
+for f in `find . -type f |egrep '\.nextflow\.log'`; do
+    echo remove $f;
+    rm $f;
+done
+