Commit for release

.github/workflows/main.yml

@@ -39,6 +39,11 @@ jobs:
     - uses: actions/checkout@v3
     - name: build and run tests
       run: |
+        sudo mkdir -p /mnt/data-volume/web_pgdata/_data
+        sudo mkdir -p /mnt/data-volume/web_prometheus/_data
+        sudo mkdir -p /mnt/data-volume/web_loki/_data
+        sudo mkdir -p /mnt/data-volume/web_grafana-storage/_data
+
         docker build -t ${{ secrets.DOCKER_USERNAME }}/minitwit-elixir-test . -f docker/minitwit-elixir-test/Dockerfile
         docker-compose up --abort-on-container-exit test
         
@@ -77,15 +82,3 @@ jobs:
           git status
           docker-compose pull web
           docker stack deploy --compose-file=docker-compose.yml web
-    # name: deploy new version
-    # runs-on: ubuntu-latest
-    # needs: build
-    # steps:
-    # - name: deploy script
-    #   run: |
-    #     ssh -o "StrictHostKeyChecking no" -i ${{ secrets.DO_SSH_KEY }} ${{ secrets.MT_USER }}@${{ secrets.MT_SERVER }}
-    #     cd itu-devops
-    #     git pull
-    #     git status
-    #     docker-compose pull web
-    #     docker-compose up -d web

.gitignore

@@ -38,3 +38,8 @@ npm-debug.log
 *.iml
 /Unikey.pem
 .vagrant
+
+secrets
+ssh_key/*
+temp/
+.terraform.lock.hcl

docker-compose.yml

@@ -19,7 +19,7 @@ services:
       PGDATABASE: minitwit_elixir_dev
       PGPORT: 5432
       # Hostname of our Postgres container.
-      PGHOST: 134.209.234.174
+      PGHOST: 157.245.21.206
       # Use the docker mix env
       MIX_ENV: docker
     deploy:
@@ -121,9 +121,41 @@ services:
           - "node.role==manager" # This makes sure it is run on the manager
 
 
+  visualizer:
+    image: dockersamples/visualizer:stable
+    ports:
+      - "8888:8080"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    deploy:
+      placement:
+        constraints:
+          - "node.role==manager" # This makes sure it is run on the manager
+
+
 # Define the volumes.
 volumes:
   pgdata:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '/mnt/data-volume/web_pgdata/_data'
   prometheus:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '/mnt/data-volume/web_prometheus/_data'
   loki:
-  grafana-storage:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '/mnt/data-volume/web_loki/_data'
+  grafana-storage:
+    driver: local
+    driver_opts:
+      type: 'none'
+      o: 'bind'
+      device: '/mnt/data-volume/web_grafana-storage/_data'

docker/minitwit-elixir/Dockerfile

@@ -14,9 +14,9 @@ RUN mix local.hex --force && \
 WORKDIR /app
 
 RUN mix deps.get
+RUN mix compile
 
 EXPOSE 4000
 
 RUN chmod +x /app/entrypoint.sh
 ENTRYPOINT ["/app/entrypoint.sh"]
-

entrypoint.sh

@@ -9,8 +9,6 @@ done
 # Create, migrate, and seed database if it doesn't exist.
 if [[ -z $(psql -Atqc "\\list $PGDATABASE") ]]; then
   echo "Database $PGDATABASE does not exist. Creating..."
-  mix deps.get
-  mix compile
   mix ecto.create
   echo "Database $PGDATABASE created."
 fi

terraform/backend.tf

@@ -0,0 +1,8 @@
+terraform {
+  backend "s3" {
+    region = "us-west-1"
+    endpoint = "https://fra1.digitaloceanspaces.com"
+    skip_credentials_validation = true
+    skip_metadata_api_check = true
+  }
+}

terraform/bootstrap.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+echo -e "\n--> Bootstrapping Minitwit\n"
+
+echo -e "\n--> Loading environment variables from secrets file\n"
+source ../secrets
+
+echo -e "\n--> Checking that environment variables are set\n"
+# check that all variables are set
+[ -z "$TF_VAR_do_token" ] && echo "TF_VAR_do_token is not set" && exit
+[ -z "$SPACE_NAME" ] && echo "SPACE_NAME is not set" && exit
+[ -z "$STATE_FILE" ] && echo "STATE_FILE is not set" && exit
+[ -z "$AWS_ACCESS_KEY_ID" ] && echo "AWS_ACCESS_KEY_ID is not set" && exit
+[ -z "$AWS_SECRET_ACCESS_KEY" ] && echo "AWS_SECRET_ACCESS_KEY is not set" && exit
+
+echo -e "\n--> Initializing terraform\n"
+# initialize terraform
+terraform init \
+    -backend-config "bucket=$SPACE_NAME" \
+    -backend-config "key=$STATE_FILE" \
+    -backend-config "access_key=$AWS_ACCESS_KEY_ID" \
+    -backend-config "secret_key=$AWS_SECRET_ACCESS_KEY"
+
+# check that everything looks good
+echo -e "\n--> Validating terraform configuration\n"
+terraform validate
+
+# See the terraform plan
+terraform plan
+
+# create infrastructure
+echo -e "\n--> Creating Infrastructure\n"
+terraform apply -auto-approve
+
+# deploy the stack to the cluster
+#echo -e "\n--> Deploying the Minitwit stack to the cluster\n"
+ssh \
+    -o 'StrictHostKeyChecking no' \
+    root@$(terraform output -raw minitwit-swarm-leader-ip-address) \
+    -i ../ssh_key/terraform \
+    'cd itu-devops && git pull && docker stack deploy --compose-file docker-compose.yml web'
+
+echo -e "\n--> Done bootstrapping Minitwit"
+echo -e "--> The dbs and elixir will need a moment to initialize, this can take up to a couple of minutes..."
+echo -e "--> Site will be available @ http://$(terraform output -raw public_ip):4000"
+echo -e "--> You can check the status of swarm cluster @ http://$(terraform output -raw minitwit-swarm-leader-ip-address):8888"
+echo -e "--> ssh to swarm leader with 'ssh root@\$(terraform output -raw minitwit-swarm-leader-ip-address) -i ssh_key/terraform'"
+echo -e "--> To remove the infrastructure run: terraform destroy -auto-approve"

terraform/ip.tf

@@ -0,0 +1,8 @@
+resource "digitalocean_floating_ip_assignment" "public-ip" {
+  ip_address = "157.245.21.206"
+  droplet_id = digitalocean_droplet.minitwit-swarm-leader.id
+}
+
+output "public_ip" {
+  value = "157.245.21.206"
+}

terraform/minitwit.auto.tfvars

@@ -0,0 +1,6 @@
+# do region
+region = "fra1"
+
+# ssh key
+pub_key = "../ssh_key/terraform.pub"
+pvt_key = "../ssh_key/terraform"

terraform/minitwit.tf

@@ -0,0 +1,116 @@
+
+#  _                _
+# | | ___  __ _  __| | ___ _ __
+# | |/ _ \/ _` |/ _` |/ _ \ '__|
+# | |  __/ (_| | (_| |  __/ |
+# |_|\___|\__,_|\__,_|\___|_|
+
+# create cloud vm
+resource "digitalocean_droplet" "minitwit-swarm-leader" {
+  image = "docker-18-04"
+  name = "minitwit-swarm-leader"
+  region = var.region
+  size = "s-2vcpu-4gb-amd"
+  # add public ssh key so we can access the machine
+  ssh_keys = [digitalocean_ssh_key.minitwit.fingerprint]
+
+  # specify a ssh connection
+  connection {
+    user = "root"
+    host = self.ipv4_address
+    type = "ssh"
+    private_key = file(var.pvt_key)
+    timeout = "2m"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      # Clone git repo for docker com
+      "git clone https://github.com/itu-devops2022/itu-devops.git",
+      # allow ports for docker swarm
+      "ufw allow 2377/tcp",
+      "ufw allow 7946",
+      "ufw allow 4789/udp",
+      # ports for apps
+      "ufw allow 4000",
+      "ufw allow 5432",
+      "ufw allow 3000",
+      "ufw allow 9090",
+      "ufw allow 3100",
+
+      # make dir for storing temp worker token
+      "mkdir -p temp",
+
+      # initialize docker swarm cluster
+      "docker swarm init --advertise-addr ${self.ipv4_address}",
+    ]
+  }
+
+  # save the worker join token
+  provisioner "local-exec" {
+    command = "ssh -o 'StrictHostKeyChecking no' root@${self.ipv4_address} -i ../ssh_key/terraform 'docker swarm join-token worker -q' > temp/worker_token"
+  }
+}
+
+
+#                     _
+# __      _____  _ __| | _____ _ __
+# \ \ /\ / / _ \| '__| |/ / _ \ '__|
+#  \ V  V / (_) | |  |   <  __/ |
+#   \_/\_/ \___/|_|  |_|\_\___|_|
+#
+# create cloud vm
+resource "digitalocean_droplet" "minitwit-swarm-worker" {
+  # create workers after the leader
+  depends_on = [digitalocean_droplet.minitwit-swarm-leader]
+
+  # number of vms to create
+  count = 1
+
+  image = "docker-18-04"
+  name = "minitwit-swarm-worker-${count.index}"
+  region = var.region
+  size = "s-2vcpu-4gb-amd"
+  # add public ssh key so we can access the machine
+  ssh_keys = [digitalocean_ssh_key.minitwit.fingerprint]
+
+  # specify a ssh connection
+  connection {
+    user = "root"
+    host = self.ipv4_address
+    type = "ssh"
+    private_key = file(var.pvt_key)
+    timeout = "2m"
+  }
+
+  provisioner "file" {
+    source = "temp/worker_token"
+    destination = "/root/worker_token"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      # allow ports for docker swarm
+      "ufw allow 2377/tcp",
+      "ufw allow 7946",
+      "ufw allow 4789/udp",
+      # ports for apps
+      "ufw allow 4000",
+      "ufw allow 5432",
+      "ufw allow 3000",
+      "ufw allow 9090",
+      "ufw allow 3100",
+
+      # join swarm cluster as workers
+      "docker swarm join --token $(cat worker_token) ${digitalocean_droplet.minitwit-swarm-leader.ipv4_address}"
+    ]
+  }
+}
+
+output "minitwit-swarm-leader-ip-address" {
+  value = digitalocean_droplet.minitwit-swarm-leader.ipv4_address
+}
+
+output "minitwit-swarm-worker-ip-address" {
+  value = digitalocean_droplet.minitwit-swarm-worker.*.ipv4_address
+}

terraform/provider.tf

@@ -0,0 +1,29 @@
+# api token
+# here it is exported in the environment like
+# export TF_VAR_do_token=xxx
+variable "do_token" {}
+
+# do region
+variable "region" {}
+
+# make sure to generate a pair of ssh keys
+variable "pub_key" {}
+variable "pvt_key" {}
+
+# setup the provider
+terraform {
+  required_providers {
+    digitalocean = {
+      source = "digitalocean/digitalocean"
+      version = "~> 2.8.0"
+    }
+    null = {
+      source = "hashicorp/null"
+      version = "3.1.0"
+    }
+  }
+}
+
+provider "digitalocean" {
+  token = var.do_token
+}

terraform/ssh_key.tf

@@ -0,0 +1,5 @@
+# add the ssh key
+resource "digitalocean_ssh_key" "minitwit" {
+  name = "minitwit"
+  public_key = file(var.pub_key)
+}

terraform/volume.tf

@@ -0,0 +1,26 @@
+data "digitalocean_volume" "data-volume" {
+  name   = "data-volume"
+}
+
+resource "digitalocean_volume_attachment" "data-volume" {
+  droplet_id = digitalocean_droplet.minitwit-swarm-leader.id
+  volume_id  = data.digitalocean_volume.data-volume.id
+
+  # create a connection to the leader
+  connection {
+    user = "root"
+    host = digitalocean_droplet.minitwit-swarm-leader.ipv4_address
+    type = "ssh"
+    private_key = file(var.pvt_key)
+    timeout = "2m"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "mkdir -p /mnt/data-volume",
+      "mount -o discard,defaults,noatime /dev/disk/by-id/scsi-0DO_Volume_data-volume /mnt/data-volume",
+      "cd /mnt",
+      "chmod -R 777 data-volume",
+    ]
+  }
+}