From 5f1db2c199b3fd0777bc99f1873c87630bd2a05b Mon Sep 17 00:00:00 2001 From: Hugh Saunders Date: Tue, 16 Apr 2019 08:54:37 +0100 Subject: [PATCH] Initialise bash-lib repo This commit adds the structure of the libraries and a few funcions. Most of the code in this commit is geared towards making sure this repo stays tested and documented in future. The following things are checked: * All functions are documented * All functions are tested * All scripts are linted See Readme for a list of the functions that are included, and for more information on testing with BATS. --- .gitmodules | 0 .gittrees | 13 + Jenkinsfile | 47 +++ README.md | 203 +++++++++++- filehandling/lib | 24 ++ git/lib | 171 ++++++++++ helpers/lib | 20 ++ init | 37 +++ k8s/Dockerfile | 33 ++ k8s/lib | 55 ++++ k8s/platform_login | 23 ++ logging/lib | 13 + run-tests | 16 + secrets.yml | 9 + test-utils/lib | 52 +++ test-utils/tap2junit/Dockerfile | 4 + test-utils/tap2junit/constraints.txt | 2 + test-utils/tap2junit/requirements.txt | 1 + test-utils/tap2junit/tap2junit.py | 91 +++++ tests-for-this-repo/filehandling.bats | 16 + .../fixtures/test-utils/tap2junit.in | 6 + .../fixtures/test-utils/tap2junit.out | 11 + tests-for-this-repo/git.bats | 310 ++++++++++++++++++ tests-for-this-repo/helpers.bats | 35 ++ tests-for-this-repo/k8s.bats | 27 ++ tests-for-this-repo/lint.bats | 69 ++++ tests-for-this-repo/logging.bats | 10 + tests-for-this-repo/python-lint/Dockerfile | 7 + .../python-lint/constraints.txt | 13 + .../python-lint/requirements.txt | 1 + tests-for-this-repo/run-bats-tests | 69 ++++ tests-for-this-repo/run-gitleaks | 18 + tests-for-this-repo/run-python-lint | 20 ++ tests-for-this-repo/test-utils.bats | 80 +++++ 34 files changed, 1504 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 100644 .gittrees create mode 100644 Jenkinsfile create mode 100644 filehandling/lib create mode 100644 git/lib create mode 100644 helpers/lib create mode 100644 init create mode 100644 k8s/Dockerfile create mode 100644 k8s/lib create mode 100755 k8s/platform_login create mode 100644 logging/lib create mode 100755 run-tests create mode 100644 secrets.yml create mode 100644 test-utils/lib create mode 100644 test-utils/tap2junit/Dockerfile create mode 100644 test-utils/tap2junit/constraints.txt create mode 100644 test-utils/tap2junit/requirements.txt create mode 100755 test-utils/tap2junit/tap2junit.py create mode 100644 tests-for-this-repo/filehandling.bats create mode 100644 tests-for-this-repo/fixtures/test-utils/tap2junit.in create mode 100644 tests-for-this-repo/fixtures/test-utils/tap2junit.out create mode 100644 tests-for-this-repo/git.bats create mode 100644 tests-for-this-repo/helpers.bats create mode 100755 tests-for-this-repo/k8s.bats create mode 100755 tests-for-this-repo/lint.bats create mode 100644 tests-for-this-repo/logging.bats create mode 100644 tests-for-this-repo/python-lint/Dockerfile create mode 100644 tests-for-this-repo/python-lint/constraints.txt create mode 100644 tests-for-this-repo/python-lint/requirements.txt create mode 100755 tests-for-this-repo/run-bats-tests create mode 100755 tests-for-this-repo/run-gitleaks create mode 100755 tests-for-this-repo/run-python-lint create mode 100644 tests-for-this-repo/test-utils.bats diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e69de29 diff --git a/.gittrees b/.gittrees new file mode 100644 index 0000000..020bd5c --- /dev/null +++ b/.gittrees @@ -0,0 +1,13 @@ +# Git Subtrees + +# The advantage of subtrees is that users don't have to care about them - its +# just a single repo. The disadvantage is that git doesn't track the metadata +# as it does for submodules. + +# This file provides an enumeration of the subtrees in this repo, and the URLs +# they came from. + +# subtree_path remote_url remote_name +test-utils/bats https://github.com/bats-core/bats bats +test-utils/bats-support https://github.com/ztombol/bats-support bats-support +test-utils/bats-assert-1 https://github.com/jasonkarns/bats-assert-1 bats-assert-1 diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..61c37d3 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,47 @@ +#!/usr/bin/env groovy + +pipeline { + agent { label 'executor-v2' } + + options { + timestamps() + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + triggers { + cron(getDailyCronString()) + } + + environment { + BATS_OUTPUT_FORMAT="junit" + } + + stages { + + stage('BATS Tests') { + steps { + sh './tests-for-this-repo/run-bats-tests' + } + } + + stage('Python Linting') { + steps { + sh './tests-for-this-repo/python-lint' + } + } + + stage('Secrets Leak Check') { + steps { + sh './tests-for-this-repo/run-gitleaks' + } + } + + } + + post { + always { + junit '*-junit.xml' + cleanupAndNotify(currentBuild.currentResult) + } + } +} diff --git a/README.md b/README.md index d8aa8e3..eb8dd01 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,201 @@ -# bashlib -Common bash functions for use in test pipelines +# bash-lib +``` + _______________ _______________ + .' .' .| + .' .' .' | + .'_______________.'______________ .' | + | ___ _____ ___ || ___ _____ ___ | | + ||_=_|__=__|_=_||||_=_|__=__|_=_|| | + ______||_____===_____||||_____===_____|| | __________ + .' ||_____===_____||||_____===_____|| .' .'| + .' ||_____===_____||||_____===_____|| .' .' | +.'___________|_______________||_______________|.'__________.' | +|.----------.|.-----___-----.||.-----___-----.|| |_____.----------. +|] |||_____________||||_____________||| .' [ | +|| ||.-----___-----.||.-----___-----.||.' | | +|| |||_____________||||_____________|||==========| | +|| ||.-----___-----.||.-----___-----.|| |_____| | +|] o|||_____________||||_____________||| .' [ 'o| +|| ||.-----___-----.||.-----___-----.||.' | | +|| ||| ||||_____________|||==========| | +|| ||| |||.-----___-----.|| |_____| | +|] ||| |||| ||| .' [ | +||__________|||_____________||||_____________|||.'________|__________| +''----------'''------------------------------'''----------'' + (o)LGB (o) +``` + +The place to store functions that are used in pipelines for multiple repos. + +Please add whatever is useful to you, but keep it tidy so its still useful to everyone else :) + +## Usage + +Add bash-lib into your project in the way that best fits your workflow. The only requirement is that you **pin the version of +bash-lib that you use**. This is important so that changes to bash-lib do not have the power to break all projects that use +bash-lib. Your project can then test updates to bash-lib and roll forward periodicly. + +Options: +* Add a submodule: they are an easy way to integrate bash-lib and automatically use a single SHA until manually updated. Submodules add a pointer from a mount point in your repo to the external repo (bash-lib), and require workflow changes to ensure that pointer is derferenced during clone, checkout and some other opertaions. +* Add a subtree: This repo uses subtrees to pull in test dependencies. Various helpers for working with subtrees can be found in [git/lib](git/lib). Subtrees copy an external repo into a subdirectory of the host repo, no workflow changes are required. Tools are provided for updating the subtree. Subtrees naturally keep a single version of bash-lib until explicitly updated. Note that subtree merge commits do not rebase well :warning:, so best to keep subtree updates in separate PRs from normal commits. +* Clone bash-lib in your deployment process, bash-lib doesn't have to be within your repo, just needs to be somewhere where your scripts can source [init](init). This is where it's most important that you implement a mechanism to always use the same SHA, as a **clone will track master by default, which is not an allowed use of bash-lib**. + +Once you have bash-lib cloned in your project, you source two things: + +1. Source `bash-lib/init`. This ensures submodules are initalised and sets the BASH_LIB_DIR env var to the absolute path to the bash-lib dir. This makes it easy to source libraries from other scripts. +2. Source `${BASH_LIB_DIR}/lib-name/lib` for any libraries you are interested in. + +You are now ready to use bash-lib functions :) + +## Structure +The `/init` script sets up everything required to use the library, most +importantly the `BASH_LIB_DIR` variable which gives the absolute path to the root +of the library and should be used for sourcing the modules. + +The repo is organized into libraries, each library is a directory that has a +lib file. Sourcing the lib for a library should expose all the functions +that library offers. The lib file may source or reference other supporting +files within it's directory. + +``` +. +├── libname +│ ├── lib +│ └── supporting-file +├── init # init script, source this first +├── run-tests # top level test script, executes all tests +├── secrets.yml # secrets required for executing tests +├── test-utils +│ ├── bats # subtree +│ ├── bats-assert-1 # subtree +│ ├── bats-support # subtree +│ ├── lib +│ └── tap2junit +└── tests-for-this-repo + ├── filehandling.bats + ├── fixtures # + │ └── libname # Dir containing test fixtures for a library + ├── tap2junit + ├── libname.bats # contains tests for libname/lib + ├── python-lint # supporting files for python lint + ├── run-bats-tests # script to run bats tests + ├── run-gitleaks # script to check for leaked secrets + └── run-python-lint # script to run python lint +``` +## Style Guide +Follow the [google shell style guide](https://google.github.io/styleguide/shell.xml#Naming_Conventions). +TL;DR: +1. Use snake_case function and variable names +1. Use `function` when declaring functions. + + +## Contents + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LibraryDescriptionFunctions
filehandlingFunctions relating to file and path handling + +
    +
  1. abs_path: Ensure a path is absolute
    2. +
+
gitGit helpers +
    +
  1. repo_root: Find the root of the current git repo.
    2. +
  2. all_files_in_repo: List files tracked by git.
    3. +
  3. tracked_files_excluding_subtrees: List files tracked by git, but excluding any files that are in paths listed in .gittrees.
    4. +
  4. cat_gittrees: Returns the contents of .gittrees from the top level of the repo, excluding any comments. Fails if .gittrees is not present.
    5. +
  5. ensure_gittrees: Creates git trees with header comments if it doesn't already exist.
    6. +
  6. add_subtree: Adds a subtree to the current repo, and stores the path, remote_url and remote_name in .gittrees so that this subtree can be maniuplated in future.
    7. +
  7. add_gittree_remotes: Read .gittrees and add git remotes for each remote_url and remote_name pair.
    8. +
  8. update_subtree: Changes a subtree to reflect a different ref in the upstream repo. Checks if the new ref is the same as the current ref before proceeding.
    9. +
  9. remote_latest_tag: Returns the symbolic name of the latest tag from a remote.
    10. +
  10. remote_latest_tagged_commit: Returns the SHA of the most recently tagged commit in a remote repo (tag^{}).
    11. +
  11. remote_sha_for_ref: Returns the SHA for a given ref from a named remote.
    12. +
  12. remote_tag_for_sha: Returns the tag corresponding to a SHA from a named remote - if there is one.
    13. +
  13. update_subtrees_to_latest_tag: Read .gittrees and update all listed subtrees to the newest tag from their upstream repo.
    14. +
+
helpersBash scripting helpers +
    +
  1. die: print message and exit 1
    2. +
  2. spushd/spopd: Safe verisons of pushd & popd that call die if the push/pop fails, they also drop stdout.
    3. +
+
k8sUtils for connecting to K8s +
    +
  1. build_gke_image: Build docker image for running kubectl commands against GKE.
    2. +
  2. delete_gke_image: Delete image from GKE.
    3. +
  3. run_docker_gke_command: Run command in gke-utils container, already authenticated to k8s cluster.
    4. +
+
loggingHelpers related to login +
    +
  1. announce: Echo message in ascii banner to distinguish it from other log messages.
    2. +
+
test-utilsHelpers for executing tests +
    +
  1. shellcheck_script: Execute shellcheck against a script, uses docker.
    2. +
  2. find_scripts: Find git tracked files with extension.
    3. +
  3. tap2junit: Convert a subset of TAP to JUnit XML. Retains logs for errors.
    4. +
+
+ +## Testing +Tests are written using [BATS](https://github.com/bats-core/bats). Each libould have a `lib-name.bats` file in [tests-for-this-repo](/tests-for-this-repo). +Asserts are provided by [bats-assert-1](https://github.com/jasonkarns/bats-assert-1). The value in these is that they provide useful debugging output when the assertion fails, eg expected x got y. + +Example: +```bash +# source support and assert libraries +. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" +. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" + +# source the library under test +. "${BASH_LIB_DIR}/git/lib" + +# define a test that calls a library function +@test "it does the thing" { + some_prep_work + # run is a wrapper that catches failures so that assertsions can be run, + # otherwise the test would immediately fail. + run does_the_thing + assert_success + assert_output "thing done" +} +``` + +Test fixtures should go in /tests-for-this-repo/[fixtures](tests-for-this-repo/fixtures)/lib-name. \ No newline at end of file diff --git a/filehandling/lib b/filehandling/lib new file mode 100644 index 0000000..4cd3f47 --- /dev/null +++ b/filehandling/lib @@ -0,0 +1,24 @@ +#!/bin/bash + +: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" +. "${BASH_LIB_DIR}/helpers/lib" + +#https://stackoverflow.com/a/23002317 +function abs_path() { + # generate absolute path from relative path + # $1 : relative filename + # return : absolute path + if [ -d "$1" ]; then + # dir + (spushd "$1"; pwd) + elif [ -f "$1" ]; then + # file + if [[ $1 = /* ]]; then + echo "$1" + elif [[ $1 == */* ]]; then + echo "$(spushd "${1%/*}"; pwd)/${1##*/}" + else + echo "$(pwd)/$1" + fi + fi +} \ No newline at end of file diff --git a/git/lib b/git/lib new file mode 100644 index 0000000..7194f20 --- /dev/null +++ b/git/lib @@ -0,0 +1,171 @@ +#!/bin/bash + +: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" +. "${BASH_LIB_DIR}/helpers/lib" + +# Get the top level of a git repo +function repo_root(){ + git rev-parse --show-toplevel +} + +# List files tracked by git +function all_files_in_repo(){ + git ls-tree -r HEAD --name-only +} + +# List files tracked by git, excluding paths +# listed as subtrees in /.gittrees +function tracked_files_excluding_subtrees(){ + subtrees="$(cat_gittrees | awk '{print $1}' | paste -sd '|' -)" + all_files_in_repo | grep -E -v "${subtrees}" +} + +#### +# Subtree Reference: +# https://github.com/git/git/blob/master/contrib/subtree/git-subtree.txt +#### + +# Read the .gittrees file from the root of a repo, +# Die with message describing the format of the file +# if it doesn't exist. +function cat_gittrees(){ + local -r git_trees="$(repo_root)/.gittrees" + local -r subtrees_file_format=".gittrees should contain one subtree per line,\ +space seperated with three fields: subtree_path renmote_url remote_name" + [[ -e "${git_trees}" ]] || die ".gittrees file ${git_trees} not found. ${subtrees_file_format}" + grep -E -v '^\s*$|^\s*#' "$(repo_root)/.gittrees" +} + +function ensure_gittrees(){ + local -r gittrees="$(repo_root)/.gittrees" + if [[ -e "${gittrees}" ]]; then + echo "${gittrees} already exists" + else + cat > "${gittrees}" <> "${gittrees}" + echo "Added Subtree ${subtree_prefix} to ${gittrees}" + add_gittree_remotes + git subtree add --squash --prefix "${subtree_prefix}" "${remote_name}" "${ref}" + echo "Merged ${remote_url}@${ref} to ${subtree_prefix}" + fi +} + +# Read .gittrees and add git remote for each subtree +function add_gittree_remotes(){ + cat_gittrees | while read -r _ url remote + do git remote rm "${remote}" &>/dev/null || true + git remote add -f "${remote}" "${url}" + done +} + +# Update a subtree to a new ref +function update_subtree(){ + local -r subtree_prefix="${1}" + local -r remote_name="${2}" + local -r remote_ref="${3}" + + local -r current_subtree_sha="$(git subtree split --prefix "${subtree_prefix}")" + local -r remote_sha="$(remote_sha_for_ref "${remote_name}" "${remote_ref}")" + + if [[ "${current_subtree_sha}" == "${remote_sha}" ]]; then + echo "${subtree_prefix}@${current_subtree_sha} is up to date with ${remote_url}@${remote_sha} (${remote_ref}), not updating." + else + echo "${subtree_prefix}@${current_subtree_sha} does not match ${remote_url}@${remote_sha} (${remote_ref}), updating." + spushd "$(repo_root)" + git subtree pull \ + --squash \ + "--prefix=${subtree_prefix}" \ + "${remote_name}" \ + "${remote_ref}" + spopd + fi +} + +# Find the latest tag available at a repo url +# Returns tag name, not sha +function remote_latest_tag(){ + local -r remote_url="${1}" + # In ls-remote the ^{} suffix refers to a peeled/dereferenced object. + # eg refs/tags/v0.0.1^{} shows the SHA of the commit that was tagged, + # not the SHA of the tag itself. + # Adding --refs hides peeled tags + git ls-remote --tags --refs --quiet \ + "${remote_url}" \ + | tail -n 1 \ + | cut -f 2 \ + | sed -e 's+refs/tags/++' +} + +# Find the SHA of the latests commit to be tagged in a remote repo +function remote_latest_tagged_commit(){ + local -r remote="${1}" + local -r tag="$(remote_latest_tag "${remote}")" + git ls-remote "${remote}" | awk "/refs\/tags\/${tag}\^/{print \$1}" +} + +function remote_sha_for_ref(){ + local -r remote="${1}" + local -r ref="${2}" + + # First try adding ^{} to the ref, incase it's a tag + # and needs peeling. If nothing is found for that, + # try without. + peeled_ref=$( + git ls-remote "${remote}" \ + | awk "/${ref}\^{}/{print \$1}" + ) + + if [[ -n "${peeled_ref}" ]]; then + echo "${peeled_ref}" + else + git ls-remote "${remote}" \ + | awk "/${ref}/{print \$1}" + fi +} + +function remote_tag_for_sha(){ + local -r remote="${1}" + local -r sha="${2}" + git ls-remote "${remote}" \ + | awk -F'/' "/${sha}.*tag/{ gsub(/\^{}\$/, \"\"); print \$3 }" +} + +# Read .gittrees and update all to the latest +# tag available from the upstream. +function update_subtrees_to_latest_tag(){ + add_gittree_remotes + cat_gittrees | while read -r subtree_prefix remote_url remote_name + do + local latest_remote_tag + latest_remote_tag="$(remote_latest_tag "${remote_name}")" + update_subtree "${subtree_prefix}" "${remote_name}" "${latest_remote_tag}" + done +} \ No newline at end of file diff --git a/helpers/lib b/helpers/lib new file mode 100644 index 0000000..f4cf58e --- /dev/null +++ b/helpers/lib @@ -0,0 +1,20 @@ +#!/bin/bash + +: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" + +function die(){ + echo "${@}" + exit 1 +} + +#safe pushd +function spushd(){ + if ! pushd "${1}" >/dev/null; then + die "pushd ${1} failed :(" + fi +} + +#safe popd +function spopd(){ + popd >/dev/null || die "popd failed :(" +} \ No newline at end of file diff --git a/init b/init new file mode 100644 index 0000000..c55fd63 --- /dev/null +++ b/init @@ -0,0 +1,37 @@ +#!/bin/bash + +## Initialisation Functions for the +## Conjurinc Bash Library + +# Shell Otions +set -euo pipefail + +# This script should be sourced before any of +# the other scripts in this repo. Other scripts +# make use of ${BASH_LIB_DIR} to find each other. + +# Get the relative path to the repo root +# shellcheck disable=SC2086 +BASH_LIB_DIR_RELATIVE="$(dirname ${BASH_SOURCE[0]})" + +# Must be set in order to load the filehandling +# module. Will be updated when abs_path is available. +BASH_LIB_DIR="${BASH_LIB_DIR_RELATIVE}" + +# Load the filehandling module for the abspath +# function +. "${BASH_LIB_DIR_RELATIVE}/filehandling/lib" + +# Export the absolute path +# shellcheck disable=SC2086 +BASH_LIB_DIR="$(abs_path ${BASH_LIB_DIR_RELATIVE})" +export BASH_LIB_DIR + +. "${BASH_LIB_DIR}/helpers/lib" + +# Update Submodules +spushd "${BASH_LIB_DIR}" + git submodule update --init --recursive +spopd + +export BATS_CMD="${BASH_LIB_DIR}/test-utils/bats/bin/bats" \ No newline at end of file diff --git a/k8s/Dockerfile b/k8s/Dockerfile new file mode 100644 index 0000000..f1a8e2f --- /dev/null +++ b/k8s/Dockerfile @@ -0,0 +1,33 @@ +FROM google/cloud-sdk + +ARG KUBECTL_CLI_URL + +RUN mkdir -p /src \ + /scripts +WORKDIR /src +COPY platform_login /scripts + +# Install Docker client +RUN apt-get update -y \ + && apt-get install \ + -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common \ + wget \ + && curl \ + -fsSL \ + https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg \ + | apt-key add - \ + && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") $(lsb_release -cs) stable"\ + && apt-get update \ + && apt-get install -y docker-ce \ + && rm -rf /var/lib/apt/lists/* + +# Install kubectl CLI +RUN wget -O \ + /usr/local/bin/kubectl \ + ${KUBECTL_CLI_URL:-https://storage.googleapis.com/kubernetes-release/release/v1.7.6/bin/linux/amd64/kubectl} \ + && chmod +x /usr/local/bin/kubectl \ No newline at end of file diff --git a/k8s/lib b/k8s/lib new file mode 100644 index 0000000..6bea766 --- /dev/null +++ b/k8s/lib @@ -0,0 +1,55 @@ +#!/bin/bash + +: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" +. "${BASH_LIB_DIR}/helpers/lib" + +# Sets additional required environment variables that aren't available in the +# secrets.yml file, and performs other preparatory steps +function build_gke_image() { + local image="gke-utils:latest" + local rc=0 + docker rmi ${image} || true + spushd "${BASH_LIB_DIR}/k8s" + # Prepare Docker images + docker build --tag "${image}"\ + --build-arg KUBECTL_CLI_URL="${KUBECTL_CLI_URL}" \ + . 1>&2 + rc=${?} + spopd + + return ${rc} +} + +# Delete an image from GCR, unless it is has multiple tags pointing to it +# This means another parallel build is using the image and we should +# just untag it to be deleted by the later job +function delete_gke_image() { + local image_and_tag="${1}" + + run_docker_gke_command " + gcloud container images delete --force-delete-tags -q ${image_and_tag} + " +} + +function run_docker_gke_command() { + docker run --rm \ + -i \ + -e DOCKER_REGISTRY_URL \ + -e DOCKER_REGISTRY_PATH \ + -e GCLOUD_SERVICE_KEY="/tmp${GCLOUD_SERVICE_KEY}" \ + -e GCLOUD_CLUSTER_NAME \ + -e GCLOUD_ZONE \ + -e SECRETLESS_IMAGE \ + -e KUBECTL_CLI_URL \ + -e GCLOUD_PROJECT_NAME \ + -v "${GCLOUD_SERVICE_KEY}:/tmp${GCLOUD_SERVICE_KEY}" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ~/.config:/root/.config \ + -v "${PWD}:/src" \ + -w /src \ + "gke-utils:latest" \ + bash -ec " + /scripts/platform_login + ${1} + " +} \ No newline at end of file diff --git a/k8s/platform_login b/k8s/platform_login new file mode 100755 index 0000000..ef00b5b --- /dev/null +++ b/k8s/platform_login @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +function main() { + echo Starting platform login + + gcloud auth activate-service-account \ + --key-file "${GCLOUD_SERVICE_KEY}" + + gcloud container clusters get-credentials \ + "${GCLOUD_CLUSTER_NAME}" \ + --zone "${GCLOUD_ZONE}" \ + --project "${GCLOUD_PROJECT_NAME}" + + docker login "${DOCKER_REGISTRY_URL}" \ + -u oauth2accesstoken \ + -p "$(gcloud auth print-access-token)" + + echo Platform Login Complete +} + +main diff --git a/logging/lib b/logging/lib new file mode 100644 index 0000000..76d31b5 --- /dev/null +++ b/logging/lib @@ -0,0 +1,13 @@ +#!/bin/bash + +: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" + +# Add logging functions here + +function announce() { + echo "++++++++++++++++++++++++++++++++++++++" + echo " " + echo "$@" + echo " " + echo "++++++++++++++++++++++++++++++++++++++" +} \ No newline at end of file diff --git a/run-tests b/run-tests new file mode 100755 index 0000000..f099ee4 --- /dev/null +++ b/run-tests @@ -0,0 +1,16 @@ +#!/bin/bash + +# This script is an entry point, so init +# is not assumed to have been run +# shellcheck disable=SC2086 +. "$(dirname ${BASH_SOURCE[0]})/init" +. "${BASH_LIB_DIR}/helpers/lib" + +# Run BATS Tests +"${BASH_LIB_DIR}/tests-for-this-repo/run-bats-tests" + +# Run Python Lint +"${BASH_LIB_DIR}/tests-for-this-repo/run-python-lint" + +# Run gitleaks +"${BASH_LIB_DIR}/tests-for-this-repo/run-gitleaks" diff --git a/secrets.yml b/secrets.yml new file mode 100644 index 0000000..321796f --- /dev/null +++ b/secrets.yml @@ -0,0 +1,9 @@ +KUBECTL_CLI_URL: https://storage.googleapis.com/kubernetes-release/release/v1.9.7/bin/linux/amd64/kubectl + +GCLOUD_CLUSTER_NAME: !var ci/google-container-engine-testbed/gcloud-cluster-name +GCLOUD_ZONE: !var ci/google-container-engine-testbed/gcloud-zone +GCLOUD_PROJECT_NAME: !var ci/google-container-engine-testbed/gcloud-project-name +GCLOUD_SERVICE_KEY: !var:file ci/google-container-engine-testbed/gcloud-service-key + +DOCKER_REGISTRY_URL: us.gcr.io +DOCKER_REGISTRY_PATH: us.gcr.io/conjur-gke-dev diff --git a/test-utils/lib b/test-utils/lib new file mode 100644 index 0000000..5eab41f --- /dev/null +++ b/test-utils/lib @@ -0,0 +1,52 @@ +#!/bin/bash + +: "${BASH_LIB_DIR:?BASH_LIB_DIR must be set. Please source bash-lib/init before other scripts from bash-lib.}" +. "${BASH_LIB_DIR}/git/lib" +. "${BASH_LIB_DIR}/helpers/lib" + +readonly SHELLCHECK_IMAGE="${SHELLCHECK_IMAGE:-koalaman/shellcheck}" +readonly SHELLCHECK_TAG="${SHELLCHECK_TAG:-v0.6.0}" + +# Check a single shell script for syntax +# and common errors. +function shellcheck_script(){ + # NOTE (HughSaunders): I tried using the checkstyle output of + # _shellcheck along with a checkstyle2junit xslt stylesheet + # from the shellcheck author. However Jenkins only reported + # on error per file, as the style sheet created a test element + # per file, with mulitple failure elements within. + # Jenkins expects one failure element per test. + + local -r script="${1}" + echo -e "\nChecking ${script}" + + # SC1091 - sourced scripts are not followed, ok because all scripts in the repo are found. + # SC1090 - can't follow non-constant source, ok for because all scripts are checked. + local -r ignores="-e SC1091 -e SC1090" + + if bash -n "${script}"; then + # shellcheck disable=SC2086 + docker run --rm -v "${PWD}:/mnt" ${SHELLCHECK_IMAGE}:${SHELLCHECK_TAG} ${ignores} ${script} + else + return 1 + fi +} + +function find_scripts(){ + tracked_files_excluding_subtrees \ + | while read -r file; do + grep --files-with-match '^#!.*bash' "${file}" || true + done +} + +function tap2junit(){ + local -r suite="${1:-BATS}" + + spushd "${BASH_LIB_DIR}/test-utils/tap2junit" + docker build . -t tap-junit 1>&2 + spopd + + # Run tap-junit docker image to convert BATS TAP output to Junit for consumption by jenkins + # filters stdin to stdout + docker run --rm -i tap-junit -s "${suite}" +} diff --git a/test-utils/tap2junit/Dockerfile b/test-utils/tap2junit/Dockerfile new file mode 100644 index 0000000..107fab6 --- /dev/null +++ b/test-utils/tap2junit/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.7-alpine +ENTRYPOINT python tap2junit.py +COPY requirements.txt constraints.txt tap2junit.py / +RUN pip install --no-cache-dir -r requirements.txt -c constraints.txt \ No newline at end of file diff --git a/test-utils/tap2junit/constraints.txt b/test-utils/tap2junit/constraints.txt new file mode 100644 index 0000000..9990d59 --- /dev/null +++ b/test-utils/tap2junit/constraints.txt @@ -0,0 +1,2 @@ +junit-xml==1.8 +six==1.12.0 diff --git a/test-utils/tap2junit/requirements.txt b/test-utils/tap2junit/requirements.txt new file mode 100644 index 0000000..ac04d3f --- /dev/null +++ b/test-utils/tap2junit/requirements.txt @@ -0,0 +1 @@ +junit-xml diff --git a/test-utils/tap2junit/tap2junit.py b/test-utils/tap2junit/tap2junit.py new file mode 100755 index 0000000..620c3ac --- /dev/null +++ b/test-utils/tap2junit/tap2junit.py @@ -0,0 +1,91 @@ +#!/usr/bin/python + +import os +import re +import sys +from junit_xml import TestSuite, TestCase + + +class Tap2JUnit: + """ This class reads a subset of TAP (Test Anything protocol) + and writes JUnit XML. + + Two line formats are read: + 1. (not )?ok testnum testname + 2. # diagnostic output + + 1. Starts a new test result. + 2. Adds diagnostic information to the last read result + + Any 2. lines found before a 1. line are ignored. + Any lines not matching either pattern are ignored. + + This script was written because none of the tap2junit converters + I could find inserted the failure output into the junit correctly. + And IMO a failed test with no indication of why is useless. + """ + + def __init__(self, test_suite, test_class): + self.test_suite = test_suite + self.test_class = test_class + # This Regex matches a (not) ok testnum testname line from the + # TAP specification, using named capture groups + self.result_re = re.compile( + r"^(?Pnot )?ok\s*(?P[0-9])+\s*(?P.*)$") + self.comment_re = re.compile(r"^\s*#") + self.case = None + self.cases = [] + + def process_line(self, line): + """ This funuction reads a tap stream line by line + and groups the diagnostic output with the relevant + result in a dictionary. + + Outputs a list of dicts, one for each result + """ + match = self.result_re.match(line) + if match: + # This line starts a new test result + self.case = match.groupdict() + self.case['stderr'] = [] + self.cases.append(self.case) + + return + + match = self.comment_re.match(line) + if match and self.case: + # This line contains diagnostic + # output from a failed test + self.case['stderr'].append(re.sub(r'^\s*#', '', line).rstrip()) + + def convert(self, infile=sys.stdin, out=sys.stdout): + """ Reads a subset of TAP and writes JUnit XML """ + # read lines + for line in infile.readlines(): + self.process_line(line) + + # Convert line dicts to test case objects + case_objs = [] + for case in self.cases: + case_obj = TestCase(case['testname'], self.test_class, 0, '', '') + if case['result'] == 'not ': + case_obj.add_failure_info(output="\n".join(case['stderr'])) + case_objs.append(case_obj) + + # Combine test cases into a suite + suite = TestSuite(self.test_suite, case_objs) + + # Write the suite out as XML + TestSuite.to_file(out, [suite]) + + +def main(): + t2j = Tap2JUnit( + os.environ.get('JUNIT_TEST_SUITE', 'tap2junit'), + os.environ.get('JUNIT_TEST_CLASS', 'tap2junit') + ) + t2j.convert() + + +if __name__ == "__main__": + main() diff --git a/tests-for-this-repo/filehandling.bats b/tests-for-this-repo/filehandling.bats new file mode 100644 index 0000000..1b309fb --- /dev/null +++ b/tests-for-this-repo/filehandling.bats @@ -0,0 +1,16 @@ +. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" +. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" + +. "${BASH_LIB_DIR}/filehandling/lib" + +@test "abs_path returns absolute path for PWD" { + run abs_path . + assert_output $PWD + assert_success +} + +@test "abs_path returns same path when already absolute" { + run abs_path /tmp + assert_output /tmp + assert_success +} \ No newline at end of file diff --git a/tests-for-this-repo/fixtures/test-utils/tap2junit.in b/tests-for-this-repo/fixtures/test-utils/tap2junit.in new file mode 100644 index 0000000..f05a61f --- /dev/null +++ b/tests-for-this-repo/fixtures/test-utils/tap2junit.in @@ -0,0 +1,6 @@ +1..3 +ok 1 Passed Test +not ok 2 Failed Test +# Diagnostic output +# more diagnostic output +ok 3 Another Passed Test \ No newline at end of file diff --git a/tests-for-this-repo/fixtures/test-utils/tap2junit.out b/tests-for-this-repo/fixtures/test-utils/tap2junit.out new file mode 100644 index 0000000..680ad96 --- /dev/null +++ b/tests-for-this-repo/fixtures/test-utils/tap2junit.out @@ -0,0 +1,11 @@ + + + + + + Diagnostic output + more diagnostic output + + + + diff --git a/tests-for-this-repo/git.bats b/tests-for-this-repo/git.bats new file mode 100644 index 0000000..d2e06f2 --- /dev/null +++ b/tests-for-this-repo/git.bats @@ -0,0 +1,310 @@ +. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" +. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" + +. "${BASH_LIB_DIR}/git/lib" + +# run before every test +setup(){ + local -r temp_dir="${BATS_TMPDIR}/testtemp" + local -r repo_dir="${temp_dir/}/repo" + rm -rf "${temp_dir}" + mkdir -p "${repo_dir}" + pushd ${repo_dir} + + git init + git config user.email "ci@cyberark.com" + git config user.name "Jenkins" + git commit --allow-empty -m "initial" + echo "some content" > a_file + git add a_file + git commit -a -m "some operations fail on empty repos" +} + +teardown(){ + local -r temp_dir="${BATS_TMPDIR}/testtemp" + rm -rf "${temp_dir}" +} + +@test "repo_root returns root of current repo" { + pushd ${BASH_LIB_DIR} + run repo_root + assert_output $PWD + assert_success +} + +@test "repo_root fails when not run from a git repo" { + pushd /tmp + run repo_root + assert_failure +} + +@test "all_files_in_repo lists all git tracked files" { + # untracked file shouldn't be listed in output + date > b + run all_files_in_repo + assert_output "a_file" + assert_success +} + +@test "tracked_files_excluding_subtrees excludes files in subtrees" { + run add_subtree bats "https://github.com/bats-core/bats" bats v1.0.0 + assert_success + assert [ -e bats/README.md ] + + date > untracked_file + + run tracked_files_excluding_subtrees + refute_output --partial bats + refute_output --partial untracked_file + assert_output --partial a_file + assert_success +} + +@test "cat_gittrees dies when gittrees doesn't exist" { + run cat_gittrees + assert_failure + assert_output --partial "should contain" +} + +@test "cat_gitrees skips comments" { + cat >.gittrees <> .gittrees + add_gittree_remotes + run git remote -v + assert_output --regexp 'bats.*https://github.com/bats-core/bats' +} + +@test "update_subtree doesn't add extra commits when no update is necessary" { + add_subtree bats "https://github.com/bats-core/bats" bats v1.0.0 + + HEAD="$(git rev-parse HEAD)" + assert_equal "$(git rev-list --count HEAD)" "4" + + run update_subtree bats bats v1.0.0 + assert_success + assert_output --partial "up to date" + + # HEAD should not have moved, as no update is necessary + run git rev-parse HEAD + assert_output --partial "${HEAD}" + assert_equal "$(git rev-list --count HEAD)" "4" +} + +@test "update_subtree correctly updates subtree to new ref" { + v1_1_0_sha="c706d1470dd1376687776bbe985ac22d09780327" + + add_subtree bats "https://github.com/bats-core/bats" bats v1.0.0 + HEAD="$(git rev-parse HEAD)" + assert_equal "$(git rev-list --count HEAD)" "4" + + update_subtree bats bats v1.1.0 + + run git subtree split --prefix bats + assert_output --partial "${v1_1_0_sha}" + + # HEAD should have moved as git subtree will + # have created a squash merge. + run git rev-parse HEAD + refute_output --partial "${HEAD}" + assert_equal "$(git rev-list --count HEAD)" "6" +} + +@test "remote_latest_tag gets latest tag from a remote" { + # For this test the "remote" will be local, + # because It hard to guarantee an actual remote + # won't gain new tags over time. + + date > a + git add a + git commit -m v1 + git tag -a -m v1 v1 + + date > b + git add b + git commit -m v2 + git tag -a -m v2 v2 + + run remote_latest_tag . + assert_output v2 + assert_success +} + +@test "remote_latest_tagged_commit returns sha of last tagged commit, not sha of the tag" { + date > a + git add a + git commit -m v1 + git tag -a -m v1 v1 + + date > b + git add b + git commit -m v2 + + run remote_latest_tagged_commit . + assert_output "$(git rev-parse v1^{})" + assert_success +} + +@test "remote_sha_for_ref looks up a sha for a given ref" { + git checkout -b testbranch + run remote_sha_for_ref . testbranch + assert_output "$(git rev-parse HEAD)" + assert_success +} + +@test "remote_tag_for_sha looks up a tag for a given sha" { + git tag -a -m v1 v1 + date > a + git add a + git commit -m v2 + git tag -a -m v2 v2 + + run remote_tag_for_sha . "$(git rev-parse v1^{})" + assert_output v1 + assert_success +} + +@test "update_subtrees_to_latest_tag updates subtrees to latest respective tag" { + + ## 1. Create the repo that will be added into the main repo as a subtree + + pushd ../ + mkdir -p subtree_repo + pushd subtree_repo + subtree_repo="${PWD}" + popd + popd + pushd "${subtree_repo}" + git init + git config user.email "ci@cyberark.com" + git config user.name "Jenkins" + git commit --allow-empty -m "initial" + + # commit and tag v1 + date > file + git add file + git commit -a -m "added file" + v1_sha="$(git rev-parse HEAD)" + git tag -a -m v1 v1 + + # commit and tag v2 + date > a + git add a + git commit -m v2 + v2_sha="$(git rev-parse HEAD)" + git tag -a -m v2 v2 + + # commit without tag + date > b + git add b + git commit -m notag + + popd + + + ## 2. Add the subtree repo twice, at different tags + add_subtree test_subtree_1 "${subtree_repo}" tst1_remote v1 + add_subtree test_subtree_2 "${subtree_repo}" tst2_remote v2 + + # Check subtree SHAs are as expected + assert_equal "$(git subtree split --prefix test_subtree_1)" "${v1_sha}" + assert_equal "$(git subtree split --prefix test_subtree_2)" "${v2_sha}" + + # Check commit count: two initial commits + two per subtree + assert_equal "$(git rev-list --count HEAD)" "6" + + # Check "file" exists in boths subtrees + assert [ -e test_subtree_1/file ] + assert [ -e test_subtree_2/file ] + + # Check "a" only exists in the subtree2 + assert [ ! -e test_subtree_1/a ] + assert [ -e test_subtree_2/a ] + + # Check "b" doesn't exist in either + assert [ ! -e test_subtree/b ] + assert [ ! -e test_subtree2/b ] + + + ## 3. Update subtrees + run update_subtrees_to_latest_tag + assert_success + + # Ensure only one subtree got updated + assert_output --regexp 'subtree_2@.*is up to date' + assert_output --regexp 'subtree_1@.*does not match' + + # Check subtree SHAs are as expected + assert_equal "$(git subtree split --prefix test_subtree_1)" "${v2_sha}" + assert_equal "$(git subtree split --prefix test_subtree_2)" "${v2_sha}" + + # Two commits should have been added for a single subtree update + assert_equal "$(git rev-list --count HEAD)" "8" + + # Check "file" exists in both subtrees + assert [ -e test_subtree_1/file ] + assert [ -e test_subtree_2/file ] + + # Check "a" exists in both subtrees + assert [ -e test_subtree_1/a ] + assert [ -e test_subtree_2/a ] + + # Check "b" doesn't exist in either + assert [ ! -e test_subtree/b ] + assert [ ! -e test_subtree2/b ] +} \ No newline at end of file diff --git a/tests-for-this-repo/helpers.bats b/tests-for-this-repo/helpers.bats new file mode 100644 index 0000000..af61b94 --- /dev/null +++ b/tests-for-this-repo/helpers.bats @@ -0,0 +1,35 @@ +. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" +. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" + +. "${BASH_LIB_DIR}/helpers/lib" + +@test "die exits and prints message" { + run bash -c ". ${BASH_LIB_DIR}/helpers/lib; die msg" + assert_output msg + assert_failure +} + +@test "spushd is quiet on stdout" { + run spushd /tmp + refute_output + assert_success +} + +@test "spopd is quiet on stdout" { + pushd . + run spopd + refute_output + assert_success +} + +@test "spushd dies on failure" { + run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spushd /this-doesnt-exist" + assert_output --partial "No such file or directory" + assert_failure +} + +@test "spopd dies on failure" { + run bash -c ". ${BASH_LIB_DIR}/helpers/lib; spopd" + assert_output --partial "stack empty" + assert_failure +} \ No newline at end of file diff --git a/tests-for-this-repo/k8s.bats b/tests-for-this-repo/k8s.bats new file mode 100755 index 0000000..48e1bc2 --- /dev/null +++ b/tests-for-this-repo/k8s.bats @@ -0,0 +1,27 @@ +. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" +. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" + +. "${BASH_LIB_DIR}/k8s/lib" + +@test "gke-utils image builds" { + run build_gke_image + assert_success +} + +@test "Kubernetes Cluster Is Available" { + : ${KUBECTL_CLI_URL:?Required Var, did you run tests via summon?} + run run_docker_gke_command "kubectl cluster-info" + assert_output --regexp "Kubernetes master.* is running at .*https://" + assert_success +} + +@test "Can delete gke image" { + local -r image="${DOCKER_REGISTRY_PATH}/alpine-test-${RANDOM}" + run_docker_gke_command " + docker pull alpine + docker tag alpine ${image} + docker push ${image} + " + run delete_gke_image "${image}" + assert_success +} \ No newline at end of file diff --git a/tests-for-this-repo/lint.bats b/tests-for-this-repo/lint.bats new file mode 100755 index 0000000..dab5d97 --- /dev/null +++ b/tests-for-this-repo/lint.bats @@ -0,0 +1,69 @@ +. ${BASH_LIB_DIR}/git/lib +. ${BASH_LIB_DIR}/test-utils/lib +. ${BASH_LIB_DIR}/helpers/lib + +setup() { + spushd ${BASH_LIB_DIR} +} + +# Find and check shell scripts +@test "Syntax and Shellcheck" { + FAILED="" + echo "Starting Bash Lint checks" + for script in $(find_scripts); do + shellcheck_script "${script}"\ + || FAILED="${FAILED} ${script}" + done + + [[ "${FAILED}" == "" ]] +} + +@test "Bash scripts do not have .sh suffix" { + rc=0 + for file in $(find_scripts); do + if [[ "${file}" =~ .sh$ ]]; then + # script has .sh suffix + echo "Script found with .sh suffix: ${file}, please rename" + rc=1 + fi + done + return ${rc} +} + +@test "All functions referenced in readme" { + rc=0 + for file in $(find_scripts | grep "/lib$"); do + for func_name in $(grep 'function.*()\s*{\s*$' ${file} | awk '{print $2}'| tr -dc '[a-zA-Z0-9_-\n]'); do + if ! grep -q "${func_name}" "${BASH_LIB_DIR}/README.md"; then + echo "Function ${func_name} from libriary ${file} is not mentioned in the README.md, please add a description" + rc=1 + fi + done + + if ! grep -q "${file}" "${BASH_LIB_DIR}/README.md"; then + echo "Library ${file} is not mentioned in the README.md, please add a description" + rc=1 + fi + done + return ${rc} +} + +@test "All functions tested" { + local rc=0 + for file in $(find_scripts | grep "/lib$"); do + local lib_name="$(dirname ${file})" + local bats_file="tests-for-this-repo/${lib_name}.bats" + if [[ ! -e "${bats_file}" ]]; then + echo "BATS test file ${bats_file} is missing for library ${file}" + rc=1 + else + for func_name in $(grep 'function.*()\s*{' "${file}" | awk '{print $2}'| tr -dc '[a-zA-Z0-9_\n]'); do + if ! grep -q "${func_name}" "${bats_file}"; then + echo "Function ${func_name} from libriary ${file} is not tested in ${bats_file}, please add a test." + rc=1 + fi + done + fi + done + return ${rc} +} \ No newline at end of file diff --git a/tests-for-this-repo/logging.bats b/tests-for-this-repo/logging.bats new file mode 100644 index 0000000..47bc684 --- /dev/null +++ b/tests-for-this-repo/logging.bats @@ -0,0 +1,10 @@ +. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" +. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" + +. "${BASH_LIB_DIR}/logging/lib" + +@test "announce prints all arguments" { + run announce one two one two + assert_output --partial "one two one two" + assert_success +} \ No newline at end of file diff --git a/tests-for-this-repo/python-lint/Dockerfile b/tests-for-this-repo/python-lint/Dockerfile new file mode 100644 index 0000000..5867bd0 --- /dev/null +++ b/tests-for-this-repo/python-lint/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.7-alpine + +ENTRYPOINT pytest --flake8 --junit-xml junit.xml +VOLUME ["/mnt"] +COPY requirements.txt constraints.txt / +RUN pip install --no-cache-dir -r requirements.txt -c constraints.txt +WORKDIR /mnt \ No newline at end of file diff --git a/tests-for-this-repo/python-lint/constraints.txt b/tests-for-this-repo/python-lint/constraints.txt new file mode 100644 index 0000000..ab6d3d5 --- /dev/null +++ b/tests-for-this-repo/python-lint/constraints.txt @@ -0,0 +1,13 @@ +atomicwrites==1.3.0 +attrs==19.1.0 +entrypoints==0.3 +flake8==3.7.7 +mccabe==0.6.1 +more-itertools==7.0.0 +pluggy==0.9.0 +py==1.8.0 +pycodestyle==2.5.0 +pyflakes==2.1.1 +pytest==4.4.1 +pytest-flake8==1.0.4 +six==1.12.0 diff --git a/tests-for-this-repo/python-lint/requirements.txt b/tests-for-this-repo/python-lint/requirements.txt new file mode 100644 index 0000000..5cc3adc --- /dev/null +++ b/tests-for-this-repo/python-lint/requirements.txt @@ -0,0 +1 @@ +pytest-flake8 diff --git a/tests-for-this-repo/run-bats-tests b/tests-for-this-repo/run-bats-tests new file mode 100755 index 0000000..455319e --- /dev/null +++ b/tests-for-this-repo/run-bats-tests @@ -0,0 +1,69 @@ +#!/bin/bash + +set -euo pipefail + +# This script runs the self tests for the bash-lib repo. + +# This script is an entry point, so init +# is not assumed to have been run +# shellcheck disable=SC2086,SC2046 +. $(dirname ${BASH_SOURCE[0]})/../init +. "${BASH_LIB_DIR}/test-utils/lib" +. "${BASH_LIB_DIR}/helpers/lib" + + +# Check vital tools are installed +if ! command -v summon >/dev/null; then + die "Summon must be installed and configured in order to run tests" +fi + +if ! command -v docker >/dev/null; then + die "Docker must be installed and configured in order to run tests" +fi + +# could be tap, junit or pretty +readonly BATS_OUTPUT_FORMAT="${BATS_OUTPUT_FORMAT:-pretty}" +readonly BATS_SUITE="${BATS_SUITE:-BATS}" +readonly TAP_FILE="${BASH_LIB_DIR}/bats.tap" + +# return code +rc=0 + +if [[ ${#} -eq 0 ]]; then + echo "No test scripts specified, running all." + scripts="${BASH_LIB_DIR}/tests-for-this-repo/*.bats" +else + scripts="${*}" +fi + +readonly summon_cmd="summon -f ${BASH_LIB_DIR}/secrets.yml" + +case $BATS_OUTPUT_FORMAT in + pretty) + # shellcheck disable=SC2086 + if ! ${summon_cmd} ${BATS_CMD} ${scripts}; then + rc=1 + fi + ;; + tap | junit) + # shellcheck disable=SC2086 + if ! ${summon_cmd} ${BATS_CMD} ${scripts} > ${TAP_FILE}; then + rc=1 + fi + echo "TAP Output written to ${TAP_FILE}" + ;; + *) + echo "Invalid BATS_OUTPUT_FORMAT: ${BATS_OUTPUT_FORMAT}, valid options: pretty, junit, tap." + exit 1 + ;; +esac + +#Convert TAP to Junit when required +if [[ "${BATS_OUTPUT_FORMAT}" == junit ]]; then + # Run tap2junit docker image to convert BATS TAP output to Junit for consumption by jenkins + readonly JUNIT_FILE="${BASH_LIB_DIR}/${BATS_SUITE}-junit.xml" + tap2junit < "${TAP_FILE}" > "${JUNIT_FILE}" + echo "Junit output written to ${JUNIT_FILE}" +fi + +exit ${rc} \ No newline at end of file diff --git a/tests-for-this-repo/run-gitleaks b/tests-for-this-repo/run-gitleaks new file mode 100755 index 0000000..0faf3fd --- /dev/null +++ b/tests-for-this-repo/run-gitleaks @@ -0,0 +1,18 @@ +#!/bin/bash + +# Check this repo for secret leaks + +# This script is an entry point, so init +# is not assumed to have been run +# shellcheck disable=SC2086,SC2046 +. $(dirname "${BASH_SOURCE[0]}")/../init +. "${BASH_LIB_DIR}/test-utils/lib" +. "${BASH_LIB_DIR}/helpers/lib" + +docker run \ + --rm \ + --name=gitleaks \ + -v ${BASH_LIB_DIR}:/code/ \ + zricethezav/gitleaks:v2.0.0 \ + -v \ + --repo-path=/code/ diff --git a/tests-for-this-repo/run-python-lint b/tests-for-this-repo/run-python-lint new file mode 100755 index 0000000..c14e96b --- /dev/null +++ b/tests-for-this-repo/run-python-lint @@ -0,0 +1,20 @@ +#!/bin/bash + +# This script is an entry point, so init +# is not assumed to have been run + +# shellcheck disable=SC2086,SC2046 +. $(dirname ${BASH_SOURCE[0]})/../init +. "${BASH_LIB_DIR}/helpers/lib" + +rc=0 + +spushd ${BASH_LIB_DIR}/tests-for-this-repo/python-lint + docker build . -t pytest-flake8 + if ! docker run --rm -v "${BASH_LIB_DIR}:/mnt" pytest-flake8; then + rc=1 + fi + mv "${BASH_LIB_DIR}/junit.xml" "${BASH_LIB_DIR}/python-lint-junit.xml" +spopd + +exit ${rc} \ No newline at end of file diff --git a/tests-for-this-repo/test-utils.bats b/tests-for-this-repo/test-utils.bats new file mode 100644 index 0000000..c723c34 --- /dev/null +++ b/tests-for-this-repo/test-utils.bats @@ -0,0 +1,80 @@ +. "${BASH_LIB_DIR}/test-utils/bats-support/load.bash" +. "${BASH_LIB_DIR}/test-utils/bats-assert-1/load.bash" + +. "${BASH_LIB_DIR}/helpers/lib" +. "${BASH_LIB_DIR}/test-utils/lib" + +docker_safe_tmp(){ + # neither mktemp -d not $BATS_TMPDIR + # produce dirs that docker can mount from + # in macos. + local -r tmp_dir="/tmp/${RANDOM}/spgs" + ( + rm -rf "${tmp_dir}" + mkdir -p "${tmp_dir}" + ) 1>&2 + echo "${tmp_dir}" +} + +@test "shellcheck notices compile error" { + tmp_dir="$(docker_safe_tmp)" + spushd "${tmp_dir}" + + echo "'" > bad_script + run shellcheck_script bad_script + assert_failure + assert_output --partial "syntax error" + + spopd + rm -rf "/tmp/${tmp_dir#/tmp/}" +} + +@test "shellcheck passes good script" { + tmp_dir="$(docker_safe_tmp)" + spushd "${tmp_dir}" + + echo -e "#!/bin/bash\n:" > good_script + run shellcheck_script good_script + rm -rf "${tmp_dir}" + assert_output --partial "Checking good_script" + assert_success + + spopd + rm -rf "/tmp/${tmp_dir#/tmp/}" +} + +@test "find_scripts finds git tracked files containing bash shebang" { + tmp_dir="${BATS_TMPDIR}/ffgtfwse" + rm -rf "${tmp_dir}" + mkdir -p "${tmp_dir}" + pushd ${tmp_dir} + git init + git config user.email "ci@ci.ci" + git config user.name "Jenkins" + + echo '#!/bin/bash' > a + echo '#!/bin/bash' > b + date > c + date > d + + git add a c + git commit -a -m "initial" + + run find_scripts + assert_output "a" + assert_success + popd +} + +@test "tap2junit correctly converts test file" { + rc=0 + fdir="${BASH_LIB_DIR}/tests-for-this-repo/fixtures/test-utils" + # Can't use run / assert_output here + # because assert_output uses $output + # which is a combination of stdout and stderr + # and we are only interested in stdout. + stdout=$(tap2junit < "${fdir}/tap2junit.in") + rc=${?} + assert_equal "${stdout}" "$(cat ${fdir}/tap2junit.out)" + assert_equal "${rc}" "0" +} \ No newline at end of file