Update handling of deployment / provision scripts for app servers

thegreenwebfoundation · Aug 25, 2023 · aad7bc7 · aad7bc7
1 parent 1b33087
commit aad7bc7
Show file tree

Hide file tree

Showing 9 changed files with 134 additions and 63 deletions.
diff --git a/ansible/_set_up_process_mgmt.yml b/ansible/_set_up_process_mgmt.yml
@@ -0,0 +1,44 @@
+---
+# shared steps for provisioning boxes, deploying and controlling
+# how web workers and queue workers are scaled
+- name: Set up script for running workers and gunicorn, via supervisor in project
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: deploy
+    group: deploy
+    mode: "0755"
+  loop:
+    - {
+        src: "run_worker.sh.j2",
+        dest: "{{ project_root }}/current/run_worker.sh",
+      }
+    - {
+        src: "run_gunicorn.sh.j2",
+        dest: "{{ project_root }}/current/run_gunicorn.sh",
+      }
+  become: true
+  tags:
+    - supervisor
+    - config
+
+- name: Set up supervisor entries for workers and web
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: deploy
+    group: deploy
+    mode: "0755"
+  loop:
+    - {
+        src: "supervisor.gunicorn.conf.j2",
+        dest: "/etc/supervisor/conf.d/{{ supervisor_gunicorn_app }}.conf",
+      }
+    - {
+        src: "supervisor.worker.conf.j2",
+        dest: "/etc/supervisor/conf.d/{{ supervisor_worker_job }}.conf",
+      }
+  become: true
+  tags:
+    - supervisor
+    - config
diff --git a/ansible/deploy.yml b/ansible/deploy.yml
@@ -2,11 +2,11 @@
 - name: Deploy the TGWF django admin
   hosts:
     # - app_servers
-    # - app1.thegreenwebfoundation.org
-    # - app2.thegreenwebfoundation.org
-    # - app3.thegreenwebfoundation.org
+    - app1.thegreenwebfoundation.org
+    - app2.thegreenwebfoundation.org
+    - app3.thegreenwebfoundation.org
     - app4.thegreenwebfoundation.org
-    # - hel1.thegreenwebfoundation.org
+
   remote_user: "deploy"
   become: false
 
@@ -53,43 +53,8 @@
       ansible.builtin.include_tasks: "_assemble_deploy_assets.yml"
       when: compile_assets is true
 
-    - name: Set up script for running workers and gunicorn, via supervisor
-      ansible.builtin.template:
-        src: "{{ item.src }}"
-        dest: "{{ item.dest }}"
-        owner: deploy
-        group: deploy
-        mode: "0755"
-      loop:
-        - {
-            src: "run_worker.sh.j2",
-            dest: "{{ project_root }}/current/run_worker.sh",
-          }
-        - {
-            src: "run_gunicorn.sh.j2",
-            dest: "{{ project_root }}/current/run_gunicorn.sh",
-          }
-      become: true
-      tags:
-        - supervisor
-
-    - name: Set up supervisor entries for workers and web
-      ansible.builtin.template:
-        src: "{{ item.src }}"
-        dest: "{{ item.dest }}"
-        owner: deploy
-        group: deploy
-        mode: "0755"
-      loop:
-        - {
-            src: "supervisor.gunicorn.conf.j2",
-            dest: "/etc/supervisor/conf.d/{{ tgwf_domain_name }}_web.conf",
-          }
-        - {
-            src: "supervisor.worker.conf.j2",
-            dest: "/etc/supervisor/conf.d/{{ tgwf_domain_name }}_worker.conf",
-          }
-      become: true
+    - name: Set up process management with supervisor
+      ansible.builtin.include_tasks: "_set_up_process_mgmt.yml"
       tags:
         - supervisor
 
@@ -99,6 +64,8 @@
         state: restarted
       become: true
       when: supervisor_restart is true
+      tags:
+        - supervisor
 
     - name: Reload nginx
       ansible.builtin.service:

diff --git a/ansible/inventories/prod.yml b/ansible/inventories/prod.yml
@@ -9,22 +9,26 @@ all:
       internal_ip: "10.0.0.4"
       dramatiq_threads: 2
       dramatiq_processes: 3
+    # TODO: once we have update the worker process names we can use app3 for serving production traffic
+    # letting us decommission some of the older app servers
+    app3.thegreenwebfoundation.org:
+      internal_ip: "10.0.0.6"
+      dramatiq_threads: 1
+      dramatiq_processes: 1
     app4.thegreenwebfoundation.org:
       internal_ip: "10.0.0.7"
       dramatiq_threads: 2
       dramatiq_processes: 3
 
-    hel1.thegreenwebfoundation.org:
-      internal_ip: "10.0.0.3"
-
   vars:
+    tgwf_stage: "prod"
+    tgwf_domain_name: "admin"
     project_root: "/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org"
     project_deploy_branch: "master"
-    tgwf_stage: "prod"
-    tgwf_domain_name: admin
-    ansible_user: deploy
-    supervisor_gunicorn_app: "admin_web"
-    supervisor_worker_job: "admin_worker"
+    ansible_user: "deploy"
+    supervisor_user: "deploy"
+    supervisor_gunicorn_app: "web_{{ tgwf_stage }}"
+    supervisor_worker_job: "worker_{{ tgwf_stage }}"
 
   # you can set child groups too
   children:
@@ -34,6 +38,7 @@ all:
       hosts:
         app1.thegreenwebfoundation.org:
         app2.thegreenwebfoundation.org:
+        app3.thegreenwebfoundation.org:
         app4.thegreenwebfoundation.org:
       # within the child group can define new vars which take
       # precedence over the ones further 'upstream'

diff --git a/ansible/inventories/staging.yml b/ansible/inventories/staging.yml
@@ -7,13 +7,13 @@ all:
       dramatiq_processes: 1
 
   vars:
-    project_root: "/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org"
-    project_deploy_branch: "staging"
     tgwf_stage: "staging"
     tgwf_domain_name: "admin-staging"
+    project_root: "/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org"
+    project_deploy_branch: "staging"
     ansible_user: "deploy"
-    supervisor_gunicorn_app: "web-staging"
-    supervisor_worker_job: "worker-staging"
+    supervisor_gunicorn_app: "web_{{ tgwf_stage }}"
+    supervisor_worker_job: "worker_{{ tgwf_stage }}"
 
   # you can set child groups too
   children:

diff --git a/ansible/provision_app.yml b/ansible/provision_app.yml
@@ -2,7 +2,11 @@
 - name: Provision the TGWF django web-app server
   hosts:
     # - app_servers
-    - app4.thegreenwebfoundation.org
+    # - app1.thegreenwebfoundation.org
+    # - app2.thegreenwebfoundation.org
+    # - app3.thegreenwebfoundation.org
+    # - app4.thegreenwebfoundation.org
+
   remote_user: "deploy"
   become: false
 

diff --git a/ansible/scale_processes.yml b/ansible/scale_processes.yml
@@ -0,0 +1,28 @@
+---
+- name: Control how many processes we are running without redeploying the TGWF django apps
+  hosts:
+    # - app1.thegreenwebfoundation.org
+    # - app2.thegreenwebfoundation.org
+    - app3.thegreenwebfoundation.org
+    - app4.thegreenwebfoundation.org
+  remote_user: "deploy"
+  become: false
+
+  vars:
+    # see vars in the inventory yaml files
+    # use one of: started, restarted, stopped
+    preferred_state: started
+
+  tasks:
+    - name: Set up process management with supervisor
+      ansible.builtin.include_tasks: "_set_up_process_mgmt.yml"
+      tags:
+        - supervisor
+
+    - name: Update django apps using supervisor
+      ansible.builtin.supervisorctl:
+        name: "{{ supervisor_gunicorn_app }}:"
+        state: "{{ preferred_state }}"
+      become: true
+      tags:
+        - supervisor
diff --git a/ansible/templates/supervisor.gunicorn.conf.j2 b/ansible/templates/supervisor.gunicorn.conf.j2
@@ -4,7 +4,7 @@
 [supervisord]
 environment=LC_ALL='en_US.UTF-8',LANG='en_US.UTF-8'
 
-[program:admin_web]
+[program:{{ supervisor_gunicorn_app }}]
 directory=/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org/current/
 numprocs=1
 command=bash ./run_gunicorn.sh

diff --git a/ansible/templates/supervisor.worker.conf.j2 b/ansible/templates/supervisor.worker.conf.j2
@@ -1,10 +1,8 @@
 # {{ ansible_managed }}
 # Last run: {{ template_run_date }}
-
 [supervisord]
 environment=LC_ALL='en_US.UTF-8',LANG='en_US.UTF-8'
-
-[program:admin_worker]
+[program:{{ supervisor_worker_job }}]
 directory=/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org/current/
 command=bash ./run_worker.sh
 process_name=%(process_num)02d

diff --git a/docs/deployment.md b/docs/deployment.md
@@ -20,14 +20,17 @@ This runs through the following steps:
 
 See `deploy.yml` and `deploy-workers.yml`, for more information.
 
-Assumign you have ssh access set up for the correct servers, you deploy  with the following command:
+Assuming you have ssh access set up for the correct servers, you deploy with the following command:
 
 ```
 ansible-playbook -i ansible/inventories/prod.yml ./ansible/deploy.yml
 ```
 
 Alternatively, merging code into master triggers the same ansible script via github actions.
 
+
+
+
 ### how our web servers are deployed
 
 The chart below outlines a high level model of how different moving parts serve web requests.
@@ -68,9 +71,9 @@ flowchart LR
 
     request
     request-->master
-    
+
     subgraph gunicorn
-        
+
         master-->worker1
         master-->worker2
         master-->worker3
@@ -127,7 +130,8 @@ flowchart LR
     end
 ```
 
-As the workloads we serve change, we may need to update the numbers of workers and the kinds of workers,to make the best use of the resources available to serve the workloads facing us.
+
+As the workloads we serve change, we may need to update the numbers of workers and the kinds of workers,to make the best use of the resources available to serve the workloads facing us. See "scaling processes" below for more
 
 **See the code**
 
@@ -198,7 +202,28 @@ manage.py rundramatiq --threads 1 --processes 1 --queues stats
 manage.py rundramatiq
 ```
 
-Update the number of threads and processes accordingly to allocate the appropriate amounts of  resources for the workloads.
+Update the number of threads and processes accordingly to allocate the appropriate amounts of resources for the workloads.
+
+### Scaling processes with ansible
+
+Each new deploy using the `deploy.yml` ansible playbook deploys the version of the branch specified in `project_deploy_branch`, including number of processes for both the gunicorn web server and for the dramatiq queue workers.
+
+If you only want to scale the workers up and down, and don't want to run through the whole deployment process, updating just the processes is possible.
+
+You have two possible options - first pass the `supervisor` tag to the deploy script. This will only run the steps tagged with `supervisor` in the deploy playbook.
+
+```
+ansible-playbook -i ansible/inventories/prod.yml ./ansible/deploy.yml --tags supervisor
+```
+
+Alternatively, you can run the dedicated `scale-processes.yml` playbook. This includes the same tasks as are defined in the larger `deploy` playbook:
+
+```
+ansible-playbook -i ansible/inventories/prod.yml ./ansible/deploy.yml --tags supervisor
+```
+
+These playbooks template out new scripts that supervisor the installed process monitors use to run both the gunicorn web servers and dramatiq queue workers, then send a command to update stop, start or restart these processes.
+
 
 **Further reading**