From aad7bc7692883b764821d1a0dbffe8df9a268b7d Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 25 Aug 2023 16:06:33 +0200 Subject: [PATCH] Update handling of deployment / provision scripts for app servers --- ansible/_set_up_process_mgmt.yml | 44 +++++++++++++++++ ansible/deploy.yml | 49 +++---------------- ansible/inventories/prod.yml | 21 +++++--- ansible/inventories/staging.yml | 8 +-- ansible/provision_app.yml | 6 ++- ansible/scale_processes.yml | 28 +++++++++++ ansible/templates/supervisor.gunicorn.conf.j2 | 2 +- ansible/templates/supervisor.worker.conf.j2 | 4 +- docs/deployment.md | 35 +++++++++++-- 9 files changed, 134 insertions(+), 63 deletions(-) create mode 100644 ansible/_set_up_process_mgmt.yml create mode 100644 ansible/scale_processes.yml diff --git a/ansible/_set_up_process_mgmt.yml b/ansible/_set_up_process_mgmt.yml new file mode 100644 index 00000000..93eec55d --- /dev/null +++ b/ansible/_set_up_process_mgmt.yml @@ -0,0 +1,44 @@ +--- +# shared steps for provisioning boxes, deploying and controlling +# how web workers and queue workers are scaled +- name: Set up script for running workers and gunicorn, via supervisor in project + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: deploy + group: deploy + mode: "0755" + loop: + - { + src: "run_worker.sh.j2", + dest: "{{ project_root }}/current/run_worker.sh", + } + - { + src: "run_gunicorn.sh.j2", + dest: "{{ project_root }}/current/run_gunicorn.sh", + } + become: true + tags: + - supervisor + - config + +- name: Set up supervisor entries for workers and web + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: deploy + group: deploy + mode: "0755" + loop: + - { + src: "supervisor.gunicorn.conf.j2", + dest: "/etc/supervisor/conf.d/{{ supervisor_gunicorn_app }}.conf", + } + - { + src: "supervisor.worker.conf.j2", + dest: "/etc/supervisor/conf.d/{{ supervisor_worker_job }}.conf", + } + become: true + tags: + - supervisor + - config diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 2181ae77..c8ce6b96 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -2,11 +2,11 @@ - name: Deploy the TGWF django admin hosts: # - app_servers - # - app1.thegreenwebfoundation.org - # - app2.thegreenwebfoundation.org - # - app3.thegreenwebfoundation.org + - app1.thegreenwebfoundation.org + - app2.thegreenwebfoundation.org + - app3.thegreenwebfoundation.org - app4.thegreenwebfoundation.org - # - hel1.thegreenwebfoundation.org + remote_user: "deploy" become: false @@ -53,43 +53,8 @@ ansible.builtin.include_tasks: "_assemble_deploy_assets.yml" when: compile_assets is true - - name: Set up script for running workers and gunicorn, via supervisor - ansible.builtin.template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: deploy - group: deploy - mode: "0755" - loop: - - { - src: "run_worker.sh.j2", - dest: "{{ project_root }}/current/run_worker.sh", - } - - { - src: "run_gunicorn.sh.j2", - dest: "{{ project_root }}/current/run_gunicorn.sh", - } - become: true - tags: - - supervisor - - - name: Set up supervisor entries for workers and web - ansible.builtin.template: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: deploy - group: deploy - mode: "0755" - loop: - - { - src: "supervisor.gunicorn.conf.j2", - dest: "/etc/supervisor/conf.d/{{ tgwf_domain_name }}_web.conf", - } - - { - src: "supervisor.worker.conf.j2", - dest: "/etc/supervisor/conf.d/{{ tgwf_domain_name }}_worker.conf", - } - become: true + - name: Set up process management with supervisor + ansible.builtin.include_tasks: "_set_up_process_mgmt.yml" tags: - supervisor @@ -99,6 +64,8 @@ state: restarted become: true when: supervisor_restart is true + tags: + - supervisor - name: Reload nginx ansible.builtin.service: diff --git a/ansible/inventories/prod.yml b/ansible/inventories/prod.yml index 482699b1..73b7512e 100644 --- a/ansible/inventories/prod.yml +++ b/ansible/inventories/prod.yml @@ -9,22 +9,26 @@ all: internal_ip: "10.0.0.4" dramatiq_threads: 2 dramatiq_processes: 3 + # TODO: once we have update the worker process names we can use app3 for serving production traffic + # letting us decommission some of the older app servers + app3.thegreenwebfoundation.org: + internal_ip: "10.0.0.6" + dramatiq_threads: 1 + dramatiq_processes: 1 app4.thegreenwebfoundation.org: internal_ip: "10.0.0.7" dramatiq_threads: 2 dramatiq_processes: 3 - hel1.thegreenwebfoundation.org: - internal_ip: "10.0.0.3" - vars: + tgwf_stage: "prod" + tgwf_domain_name: "admin" project_root: "/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org" project_deploy_branch: "master" - tgwf_stage: "prod" - tgwf_domain_name: admin - ansible_user: deploy - supervisor_gunicorn_app: "admin_web" - supervisor_worker_job: "admin_worker" + ansible_user: "deploy" + supervisor_user: "deploy" + supervisor_gunicorn_app: "web_{{ tgwf_stage }}" + supervisor_worker_job: "worker_{{ tgwf_stage }}" # you can set child groups too children: @@ -34,6 +38,7 @@ all: hosts: app1.thegreenwebfoundation.org: app2.thegreenwebfoundation.org: + app3.thegreenwebfoundation.org: app4.thegreenwebfoundation.org: # within the child group can define new vars which take # precedence over the ones further 'upstream' diff --git a/ansible/inventories/staging.yml b/ansible/inventories/staging.yml index 91609c73..5cdd8818 100644 --- a/ansible/inventories/staging.yml +++ b/ansible/inventories/staging.yml @@ -7,13 +7,13 @@ all: dramatiq_processes: 1 vars: - project_root: "/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org" - project_deploy_branch: "staging" tgwf_stage: "staging" tgwf_domain_name: "admin-staging" + project_root: "/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org" + project_deploy_branch: "staging" ansible_user: "deploy" - supervisor_gunicorn_app: "web-staging" - supervisor_worker_job: "worker-staging" + supervisor_gunicorn_app: "web_{{ tgwf_stage }}" + supervisor_worker_job: "worker_{{ tgwf_stage }}" # you can set child groups too children: diff --git a/ansible/provision_app.yml b/ansible/provision_app.yml index 7afbaeb6..874f2139 100644 --- a/ansible/provision_app.yml +++ b/ansible/provision_app.yml @@ -2,7 +2,11 @@ - name: Provision the TGWF django web-app server hosts: # - app_servers - - app4.thegreenwebfoundation.org + # - app1.thegreenwebfoundation.org + # - app2.thegreenwebfoundation.org + # - app3.thegreenwebfoundation.org + # - app4.thegreenwebfoundation.org + remote_user: "deploy" become: false diff --git a/ansible/scale_processes.yml b/ansible/scale_processes.yml new file mode 100644 index 00000000..f23068d8 --- /dev/null +++ b/ansible/scale_processes.yml @@ -0,0 +1,28 @@ +--- +- name: Control how many processes we are running without redeploying the TGWF django apps + hosts: + # - app1.thegreenwebfoundation.org + # - app2.thegreenwebfoundation.org + - app3.thegreenwebfoundation.org + - app4.thegreenwebfoundation.org + remote_user: "deploy" + become: false + + vars: + # see vars in the inventory yaml files + # use one of: started, restarted, stopped + preferred_state: started + + tasks: + - name: Set up process management with supervisor + ansible.builtin.include_tasks: "_set_up_process_mgmt.yml" + tags: + - supervisor + + - name: Update django apps using supervisor + ansible.builtin.supervisorctl: + name: "{{ supervisor_gunicorn_app }}:" + state: "{{ preferred_state }}" + become: true + tags: + - supervisor diff --git a/ansible/templates/supervisor.gunicorn.conf.j2 b/ansible/templates/supervisor.gunicorn.conf.j2 index 7a41f750..0a103484 100644 --- a/ansible/templates/supervisor.gunicorn.conf.j2 +++ b/ansible/templates/supervisor.gunicorn.conf.j2 @@ -4,7 +4,7 @@ [supervisord] environment=LC_ALL='en_US.UTF-8',LANG='en_US.UTF-8' -[program:admin_web] +[program:{{ supervisor_gunicorn_app }}] directory=/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org/current/ numprocs=1 command=bash ./run_gunicorn.sh diff --git a/ansible/templates/supervisor.worker.conf.j2 b/ansible/templates/supervisor.worker.conf.j2 index 514b22f8..db3c0784 100644 --- a/ansible/templates/supervisor.worker.conf.j2 +++ b/ansible/templates/supervisor.worker.conf.j2 @@ -1,10 +1,8 @@ # {{ ansible_managed }} # Last run: {{ template_run_date }} - [supervisord] environment=LC_ALL='en_US.UTF-8',LANG='en_US.UTF-8' - -[program:admin_worker] +[program:{{ supervisor_worker_job }}] directory=/var/www/{{ tgwf_domain_name }}.thegreenwebfoundation.org/current/ command=bash ./run_worker.sh process_name=%(process_num)02d diff --git a/docs/deployment.md b/docs/deployment.md index b25dba62..884ffe71 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -20,7 +20,7 @@ This runs through the following steps: See `deploy.yml` and `deploy-workers.yml`, for more information. -Assumign you have ssh access set up for the correct servers, you deploy with the following command: +Assuming you have ssh access set up for the correct servers, you deploy with the following command: ``` ansible-playbook -i ansible/inventories/prod.yml ./ansible/deploy.yml @@ -28,6 +28,9 @@ ansible-playbook -i ansible/inventories/prod.yml ./ansible/deploy.yml Alternatively, merging code into master triggers the same ansible script via github actions. + + + ### how our web servers are deployed The chart below outlines a high level model of how different moving parts serve web requests. @@ -68,9 +71,9 @@ flowchart LR request request-->master - + subgraph gunicorn - + master-->worker1 master-->worker2 master-->worker3 @@ -127,7 +130,8 @@ flowchart LR end ``` -As the workloads we serve change, we may need to update the numbers of workers and the kinds of workers,to make the best use of the resources available to serve the workloads facing us. + +As the workloads we serve change, we may need to update the numbers of workers and the kinds of workers,to make the best use of the resources available to serve the workloads facing us. See "scaling processes" below for more **See the code** @@ -198,7 +202,28 @@ manage.py rundramatiq --threads 1 --processes 1 --queues stats manage.py rundramatiq ``` -Update the number of threads and processes accordingly to allocate the appropriate amounts of resources for the workloads. +Update the number of threads and processes accordingly to allocate the appropriate amounts of resources for the workloads. + +### Scaling processes with ansible + +Each new deploy using the `deploy.yml` ansible playbook deploys the version of the branch specified in `project_deploy_branch`, including number of processes for both the gunicorn web server and for the dramatiq queue workers. + +If you only want to scale the workers up and down, and don't want to run through the whole deployment process, updating just the processes is possible. + +You have two possible options - first pass the `supervisor` tag to the deploy script. This will only run the steps tagged with `supervisor` in the deploy playbook. + +``` +ansible-playbook -i ansible/inventories/prod.yml ./ansible/deploy.yml --tags supervisor +``` + +Alternatively, you can run the dedicated `scale-processes.yml` playbook. This includes the same tasks as are defined in the larger `deploy` playbook: + +``` +ansible-playbook -i ansible/inventories/prod.yml ./ansible/deploy.yml --tags supervisor +``` + +These playbooks template out new scripts that supervisor the installed process monitors use to run both the gunicorn web servers and dramatiq queue workers, then send a command to update stop, start or restart these processes. + **Further reading**