From 59a0f565d433ba8b2fd4fc815276a2c424c6fa98 Mon Sep 17 00:00:00 2001 From: Etienne Audet-Cobello Date: Mon, 9 Sep 2024 12:48:54 -0400 Subject: [PATCH] add debug action --- .github/workflows/e2e-deleteme.yaml | 18 ++++-- hack/ci-e2e-tests.sh | 85 ++++++++++++++++++++++++----- test/e2e/config/ck8s-aws.yaml | 6 +- 3 files changed, 88 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e-deleteme.yaml b/.github/workflows/e2e-deleteme.yaml index 68464b68..984b1ad1 100644 --- a/.github/workflows/e2e-deleteme.yaml +++ b/.github/workflows/e2e-deleteme.yaml @@ -5,11 +5,12 @@ on: permissions: contents: read + id-token: write jobs: run-e2e-tests: name: Run E2E Tests - runs-on: [self-hosted, linux, X64, jammy, large] + runs-on: ubuntu-latest strategy: matrix: ginkgo_focus: @@ -19,8 +20,19 @@ jobs: #- "Workload cluster scaling" #- "Workload cluster upgrade" steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + audience: sts.amazonaws.com + aws-region: us-east-2 + role-to-assume: arn:aws:iam::018302341396:role/GithubOIDC + role-duration-seconds: 3600 - name: Check out repo uses: actions/checkout@v4 + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + detached: true - name: Install requirements run: | sudo apt install make @@ -35,7 +47,3 @@ jobs: sudo -E ./hack/ci-e2e-tests.sh true aws v0.1.2 env: GOPROXY: direct - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }} - AWS_REGION: us-east-2 diff --git a/hack/ci-e2e-tests.sh b/hack/ci-e2e-tests.sh index f193697a..1be54b94 100755 --- a/hack/ci-e2e-tests.sh +++ b/hack/ci-e2e-tests.sh @@ -1,5 +1,6 @@ #!/bin/bash +# DO NOT enable -x as it will expose sensitive information in the logs set -xe # This script is used to run e2e tests for the CK8s CAPI. @@ -20,8 +21,8 @@ readonly SKIP_CLEANUP=${1:-true} readonly INFRA_PROVIDER=${2:-aws} readonly CK8S_PROVIDER_VERSION=${3:-v0.1.2} -readonly LXD_CHANNEL="5.21/stable" -readonly LXC_IMAGE="ubuntu:20.04" +readonly LXD_CHANNEL="6.1/stable" +readonly LXC_IMAGE="ubuntu:22.04" readonly K8S_PROFILE_URL="https://github.com/raw/canonical/k8s-snap/main/tests/integration/lxd-profile.yaml" readonly K8S_PROFILE_PATH="/tmp/k8s.profile" readonly CONTAINER_NAME="k8s-test" @@ -32,8 +33,16 @@ function error_exit { return 1 } +function log_info { + printf "INFO: %s\n" "$1" +} + # Check that all required environment variables are set function check_required_env_vars { + set +x + + log_info "Checking required environment variables..." + local required_env_vars=() if [[ $INFRA_PROVIDER == "aws" ]]; then @@ -45,22 +54,42 @@ function check_required_env_vars { error_exit "Missing required environment variable: $var" fi done + + set -x } function exec_in_container { lxc exec $CONTAINER_NAME -- bash -c "$1" } +function setup_firewall { + log_info "Setting up firewall rules..." + + if sudo iptables -L DOCKER-USER; then + sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT + sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + fi +} + # Install LXD snap function install_lxd { - sudo snap install lxd --channel=$LXD_CHANNEL + log_info "Installing LXD..." + + if snap list lxd; then + sudo snap refresh lxd --channel=$LXD_CHANNEL + else + sudo snap install lxd --channel=$LXD_CHANNEL + fi + sudo lxd waitready sudo lxd init --auto sudo usermod --append --groups lxd "$USER" } # Create or ensure the k8s profile exists function setup_lxd_profile { - lxc profile create k8s || true + log_info "Setting up LXD profile..." + + lxc profile show k8s || lxc profile create k8s wget -q $K8S_PROFILE_URL -O $K8S_PROFILE_PATH cat $K8S_PROFILE_PATH | lxc profile edit k8s rm -f $K8S_PROFILE_PATH @@ -68,42 +97,60 @@ function setup_lxd_profile { # Setup and configure the container function setup_container { + log_info "Setting up LXD container..." + lxc launch $LXC_IMAGE $CONTAINER_NAME -p default -p k8s + + # Wait for container to be ready to run commands until exec_in_container true; do sleep 1 done exec_in_container "apt update && apt install -y snapd" exec_in_container "systemctl start snapd" + exec_in_container "snap wait core seed.loaded" # Script is running from the hack directory, so push the entire directory to the container - lxc file push -r .. $CONTAINER_NAME/root/ + lxc file push -r .. $CONTAINER_NAME/root/ >/dev/null } function configure_container_env { + log_info "Configuring container environment..." + + set +x + if [[ $INFRA_PROVIDER == "aws" ]]; then + log_info "Configuring AWS credentials in container..." + # Check for clusterawsadm binary exec_in_container "which clusterawsadm" || error_exit "clusterawsadm binary not found in container" - set +x lxc config set $CONTAINER_NAME environment.AWS_REGION "$AWS_REGION" lxc config set $CONTAINER_NAME environment.AWS_SECRET_ACCESS_KEY "$AWS_SECRET_ACCESS_KEY" lxc config set $CONTAINER_NAME environment.AWS_ACCESS_KEY_ID "$AWS_ACCESS_KEY_ID" - local aws_creds - aws_creds=$(lxc exec "$CONTAINER_NAME" -- bash -c "clusterawsadm bootstrap credentials encode-as-profile") + if [[ -z $AWS_SESSION_TOKEN ]]; then + log_info "AWS_SESSION_TOKEN not set. Skipping..." + else + lxc config set $CONTAINER_NAME environment.AWS_SESSION_TOKEN "$AWS_SESSION_TOKEN" + fi + # This command can fail if the stack already exists, so we ignore the error + exec_in_container "clusterawsadm bootstrap iam create-cloudformation-stack" || true + + local aws_creds=$(lxc exec "$CONTAINER_NAME" -- bash -c "clusterawsadm bootstrap credentials encode-as-profile") + echo "::add-mask::$aws_creds" # Mask the credentials in the Github CI logs. lxc config set "$CONTAINER_NAME" environment.AWS_B64ENCODED_CREDENTIALS "$aws_creds" - set -x fi + + set -x } # Main installation and configuration function setup_management_cluster { - sleep 5 - exec_in_container "snap install k8s --classic --edge" - sleep 1 - exec_in_container "snap install go --classic" + log_info "Setting up management cluster..." + exec_in_container "sudo snap install k8s --classic --edge" + exec_in_container "sudo snap install go --classic" exec_in_container "mkdir -p /root/.kube" exec_in_container "sudo k8s bootstrap" exec_in_container "sudo k8s status --wait-ready" @@ -111,12 +158,15 @@ function setup_management_cluster { } function clone_repos { + log_info "Cloning CK8s and CAPI repositories..." exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api-provider-aws /root/cluster-api-provider-aws" exec_in_container "git clone --depth 1 https://github.com/kubernetes-sigs/cluster-api /root/cluster-api" } # Transfer and execute scripts function install_tools { + log_info "Installing tools in container..." + tools=(install-clusterctl.sh) packages=(make) snaps=(kubectl) @@ -139,6 +189,8 @@ function install_tools { } function init_clusterctl { + log_info "Initializing clusterctl with $INFRA_PROVIDER infrastructure and CK8s $CK8S_PROVIDER_VERSION..." + configure_container_env # Ensures that the right environment variables are set in the container exec_in_container "chmod +x /root/cluster-api-k8s/hack/write-provider-config.sh" @@ -149,22 +201,26 @@ function init_clusterctl { } function run_e2e_tests { + log_info "Running e2e tests..." exec_in_container "cd /root/cluster-api-k8s && make USE_EXISTING_CLUSTER=true GINKGO_FOCUS=\"Workload cluster creation\" test-e2e" } function cleanup { if [[ $SKIP_CLEANUP == "true" ]]; then + log_info "Skipping cleanup..." return fi # Infra-specific cleanup if [[ $INFRA_PROVIDER == "aws" ]]; then + log_info "Cleaning up AWS resources..." exec_in_container "mkdir -p /root/.aws-nuke" exec_in_container "echo ""$AWS_NUKE_CONFIG"" > /root/.aws-nuke/config.yaml" exec_in_container "aws-nuke --config /root/.aws-nuke/config.yaml --force" fi lxc delete $CONTAINER_NAME --force + log_info "Cleanup complete." } function main { @@ -176,6 +232,7 @@ function main { check_required_env_vars install_lxd setup_lxd_profile + setup_firewall setup_container setup_management_cluster clone_repos @@ -183,6 +240,8 @@ function main { init_clusterctl run_e2e_tests cleanup + + log_info "E2E tests completed successfully." } main diff --git a/test/e2e/config/ck8s-aws.yaml b/test/e2e/config/ck8s-aws.yaml index c80c540b..1f93e009 100644 --- a/test/e2e/config/ck8s-aws.yaml +++ b/test/e2e/config/ck8s-aws.yaml @@ -32,7 +32,7 @@ providers: # By default, will use the latest version defined in ../data/shared/v1beta1/metadata.yaml # to init the management cluster - name: v2.6.1 # used during e2e-test - value: "../../../../cluster-api-provider-aws/config/default" + value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path contract: v1beta2 files: - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" @@ -46,7 +46,7 @@ providers: # default version for docker infrastructure provider # name here should match defaultProviderVersion - name: v1.9.99 - value: "../../../../cluster-api-provider-aws/config/default" + value: "../../../../cluster-api-provider-aws/config/default" # TODO don't use relative path contract: v1beta2 files: - sourcePath: "../data/shared/v1beta1_aws/metadata.yaml" @@ -95,7 +95,7 @@ variables: AWS_NODE_INSTANCE_TYPE: t3.large AWS_PUBLIC_IP: false AWS_CREATE_BASTION: true - AWS_SSH_KEY_NAME: "etienne" + AWS_SSH_KEY_NAME: "" AWS_AMI_ID: "ami-05145146e3a9db6f3" AWS_CONTROL_PLANE_ROOT_VOLUME_SIZE: 16 AWS_NODE_ROOT_VOLUME_SIZE: 16