-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
fdec4a8
commit 17e3c80
Showing
9 changed files
with
232 additions
and
55 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,57 +1,59 @@ | ||
# set the port to listen on | ||
PORT=9000 | ||
GUNICORN_BIND_IP=0.0.0.0 | ||
|
||
PYTHONDONTWRITEBYTECODE=1 | ||
PYTHONUNBUFFERED=1 | ||
PYTHONBREAKPOINT="ipdb.set_trace" | ||
PYTHONBREAKPOINT=ipdb.set_trace | ||
|
||
SECRET_KEY='some-key' | ||
SECRET_KEY=some-key | ||
|
||
DATABASE_URL=mysql://localhost:3306/greencheck | ||
DATABASE_URL_READ_ONLY=mysql://localhost:3306/greencheck | ||
# docker | ||
DATABASE_URL=mysql://deploy:deploy@db:3306/greencheck | ||
DATABASE_URL_READ_ONLY=mysql://deploy:deploy@db:3306/greencheck | ||
EXPLORER_TOKEN=some-token | ||
|
||
EXPLORER_TOKEN="some-token" | ||
MAILGUN_API_KEY="50 characters long-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" | ||
MAILGUN_API_KEY=50-characters-long-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx | ||
|
||
# used to stop defaulting to debug | ||
DJANGO_SETTINGS_MODULE=greenweb.settings.development | ||
|
||
# used for working with notebooks | ||
DJANGO_ALLOW_ASYNC_UNSAFE=True | ||
|
||
RABBITMQ_URL=amqp://guest:guest@host.docker.internal:5672/ | ||
RABBITMQ_URL=amqp://guest:guest@rabbitmq:5672/ | ||
|
||
|
||
DOMAIN_SNAPSHOT_BUCKET=tgwf-green-domains-dev | ||
OBJECT_STORAGE_INFRA_BUCKET=https://s3.nl-ams.scw.cloud | ||
# used for uploading files to object storage | ||
OBJECT_STORAGE_ENDPOINT=https://s3.nl-ams.scw.cloud | ||
OBJECT_STORAGE_REGION=nl-ams | ||
OBJECT_STORAGE_ACCESS_KEY_ID="xxxxxxxxxxxxxxxxxxxx" | ||
OBJECT_STORAGE_SECRET_ACCESS_KEY="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" | ||
OBJECT_STORAGE_BUCKET_NAME="tgwf-web-app-xxxx" | ||
OBJECT_STORAGE_ACCESS_KEY_ID=xxxxxxxxxxxxxxxxxxxx | ||
OBJECT_STORAGE_SECRET_ACCESS_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx | ||
OBJECT_STORAGE_BUCKET_NAME=tgwf-web-app-xxxx | ||
|
||
# used for geolocation of IP addresses | ||
MAXMIND_USER_ID=123456 | ||
MAXMIND_LICENCE_KEY="xxxxxxxxxxxxxxxx" | ||
MAXMIND_LICENCE_KEY=xxxxxxxxxxxxxxxx | ||
|
||
MICROSOFT_PROVIDER_ID=1234 | ||
MICROSOFT_LOCAL_FILE_DIRECTORY="https://tgwf-web-app-test.s3.nl-ams.scw.cloud/data-imports/ms-azure-ip-ranges-2022-04-25.json" | ||
MICROSOFT_LOCAL_FILE_DIRECTORY=https://tgwf-web-app-test.s3.nl-ams.scw.cloud/data-imports/ms-azure-ip-ranges-2022-04-25.json | ||
|
||
EQUINIX_PROVIDER_ID=123 | ||
EQUINIX_REMOTE_API_ENDPOINT="https://tgwf-web-app-live.s3.nl-ams.scw.cloud/data-imports/equinix.ips.and.asns.2022-02-25.txt" | ||
EQUINIX_REMOTE_API_ENDPOINT=https://tgwf-web-app-live.s3.nl-ams.scw.cloud/data-imports/equinix.ips.and.asns.2022-02-25.txt | ||
|
||
AMAZON_PROVIDER_ID=697 | ||
AMAZON_REMOTE_API_ENDPOINT="https://ip-ranges.amazonaws.com/ip-ranges.json" | ||
AMAZON_REMOTE_API_ENDPOINT=https://ip-ranges.amazonaws.com/ip-ranges.json | ||
|
||
GOOGLE_PROVIDER_ID=597 | ||
GOOGLE_DATASET_ENDPOINT="https://www.gstatic.com/ipranges/cloud.json" | ||
GOOGLE_DATASET_ENDPOINT=https://www.gstatic.com/ipranges/cloud.json | ||
|
||
|
||
# ; used to allow for convenient access to the aws cli | ||
AWS_SHARED_CREDENTIALS_FILE="/absolute/path/to/.aws.credentials" | ||
AWS_CONFIG_FILE="/absolute/path/to/.aws.config" | ||
AWS_SHARED_CREDENTIALS_FILE=/absolute/path/to/.aws.credentials | ||
AWS_CONFIG_FILE=/absolute/path/to/.aws.config | ||
|
||
API_URL="https://greenweb.localhost" | ||
API_URL=https://greenweb.localhost | ||
|
||
TRELLO_REGISTRATION_EMAIL_TO_BOARD_ADDRESS="mail-to-board@localhost" | ||
TRELLO_REGISTRATION_EMAIL_TO_BOARD_ADDRESS=mail-to-board@localhost |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,67 +1,68 @@ | ||
FROM python:3.11 as production | ||
|
||
|
||
# Update the package listing, so we know what packages exist | ||
RUN apt-get update | ||
|
||
# Install security updates: | ||
RUN apt-get upgrade --yes | ||
RUN apt-get install wget --no-install-recommends --yes | ||
RUN wget https://deb.nodesource.com/setup_18.x -O /tmp/setup_18.x.sh --no-check-certificate | ||
|
||
RUN curl https://deb.nodesource.com/setup_18.x > /tmp/setup_18.x.sh | ||
RUN bash /tmp/setup_18.x.sh | ||
RUN apt-get install nodejs --no-install-recommends --yes | ||
|
||
# Delete cached files we don't need anymore | ||
RUN apt-get clean | ||
|
||
# Delete index files we don't need anymore: | ||
RUN rm -rf /var/lib/apt/lists/* | ||
|
||
# Install dependencies in a virtualenv | ||
ENV VIRTUAL_ENV=/app/.venv | ||
|
||
RUN useradd deploy --create-home && mkdir /app $VIRTUAL_ENV && chown -R deploy /app $VIRTUAL_ENV | ||
|
||
WORKDIR /app | ||
|
||
# Set default environment variables. They are used at build time and runtime. | ||
# If you specify your own environment variables on Heroku, they will | ||
# override the ones set here. The ones below serve as sane defaults only. | ||
# * PATH - Make sure that Poetry is on the PATH, along with our venv | ||
# * PYTHONPATH - Ensure `django-admin` works correctly. | ||
# * PYTHONUNBUFFERED - This is useful so Python does not hold any messages | ||
# from being output. | ||
# https://docs.python.org/3.9/using/cmdline.html#envvar-PYTHONUNBUFFERED | ||
# https://docs.python.org/3.9/using/cmdline.html#cmdoption-u | ||
# * DJANGO_SETTINGS_MODULE - default settings used in the container. | ||
# * PORT - default port used. Please match with EXPOSE. | ||
# Adding the virtual environment to the path saves us needing to | ||
# run `source /app/.venv/bin/activate`, and adding python path | ||
# makes it easier to run manage.py commands | ||
ENV PATH=$VIRTUAL_ENV/bin:$PATH \ | ||
PYTHONPATH=/app | ||
# PYTHONUNBUFFERED=1 \ | ||
# DJANGO_SETTINGS_MODULE=greenweb.settings.production \ | ||
# PORT=9000 \ | ||
# WEB_CONCURRENCY=3 \ | ||
# GUNICORN_CMD_ARGS="-c gunicorn-conf.py --max-requests 1200 --max-requests-jitter 50 --access-logfile - --timeout 25 --reload" | ||
|
||
# Port exposed by this container. Should default to the port used by your WSGI | ||
# server (Gunicorn). Heroku will ignore this. | ||
|
||
# Default port exposed by this container | ||
EXPOSE 9000 | ||
|
||
# We don't want to use root. We use this user elsewhere without docker | ||
# so we keep the same name for consistency | ||
USER deploy | ||
|
||
# Install your app's Python requirements. | ||
# Set up our virtual env directory | ||
RUN python -m venv $VIRTUAL_ENV | ||
RUN python -m pip install uv wheel --upgrade | ||
|
||
# Add our python libraries for managing dependencies | ||
uv 0.1.43 is triggering bad certificate errors, so we pin to 0.1.39 | ||
RUN python -m pip install uv==0.1.39 wheel --upgrade | ||
|
||
# Copy application code. | ||
# Copy application code, with dockerignore filtering out the stuff we don't want | ||
# from our final build artefact | ||
COPY --chown=deploy . . | ||
|
||
|
||
# install dependencies via UV | ||
# Install dependencies via uv | ||
RUN uv pip install -r requirements/requirements.linux.generated.txt | ||
|
||
# set up front end pipeline | ||
# Set up front end pipeline | ||
RUN python ./manage.py tailwind install | ||
RUN python ./manage.py tailwind build | ||
|
||
# # # run npx rollup in correct directory | ||
# Install the other node dependencies | ||
# TODO: we might not need node in production *at all* if we can generate | ||
# the static files in the build step. Something to investigate | ||
WORKDIR /app | ||
RUN cd ./apps/theme/static_src/ && \ | ||
npx rollup --config | ||
|
||
# # TODO Collect static. This command will move static files from application | ||
# # directories and "static_compiled" folder to the main static directory that | ||
# # will be served by the WSGI server. | ||
# Collect static files | ||
RUN python ./manage.py collectstatic --noinput --clear | ||
|
||
# Use the shell form of CMD, so we have access to our environment variables | ||
# $GUNICORN_CMD_ARGS allows us to add additional arguments to the gunicorn command | ||
CMD gunicorn greenweb.wsgi --bind $GUNICORN_BIND_IP:$PORT --config gunicorn.conf.py $GUNICORN_CMD_ARGS |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -35,3 +35,4 @@ services: | |
restart: always | ||
depends_on: | ||
- db | ||
- rabbitmq |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,110 @@ | ||
# Working with Docker | ||
|
||
We use Docker to create packaged versions of the platform for use where a dockerised version of this code base is expected. | ||
|
||
The key scenarios we would use this would be | ||
|
||
- in local development with docker compose | ||
- where running code is assumed to be ephemeral, like a 'serverless' service (for example, AWS Lambda, Fly.io, Scaleway Serverless Containers, Google Cloud Run, etc.) | ||
- in tooling used to understand the resource usage of the platform, like Green Coding Solution's Green Metrics Tool (GMT) | ||
|
||
|
||
### How do use docker and docker compose for development | ||
|
||
First, make sure you have docker or docker-compatible software installed (Orbstack on MacOS is a good example of the latter). | ||
|
||
The Green Web Platform is comprised of three main services | ||
|
||
- a WSGI django application: served by the Gunicorn webserver | ||
- a message queue: RabbitMQ | ||
- a database: MariaDB | ||
|
||
This topology is represented in our Gitpod development environment, but also the Docker Compose file, `compose.yaml`, which is consumed by the Green Metrics Tool for testing runs. | ||
|
||
#### Running a local version of the full system with docker compose | ||
|
||
You can spin up a local version of the setup above with Docker Compose by checking you are in the project root directory, and calling: | ||
|
||
```shell | ||
# build the images locally, fetching the other images where needed | ||
docker compose build | ||
# run the services and main django application | ||
docker compose up | ||
``` | ||
|
||
This will download the various images needed, then start them as separate docker containers. By default the contents of the project `./apps` contianing most of the django code, and `./greenweb` directory are mounted into the running django container, allowing you to make changes. | ||
|
||
Similarly, an `.env.docker` file is used to provide the environment variables file that would be present in production, or in other development environments. See `.env.docker.sample` for an annotated list of the expected environment variables. | ||
|
||
```yml | ||
# abridged file - see compose.yaml for more details | ||
django: | ||
env_file: | ||
- path: ./.env.docker | ||
build: | ||
context: . | ||
dockerfile: Dockerfile | ||
container_name: greenweb-app | ||
image: greenweb-app | ||
expose: | ||
- 9000 | ||
ports: | ||
- 9000:9000 | ||
volumes: | ||
- ./apps:/app/apps | ||
- ./greenweb:/app/greenweb | ||
restart: always | ||
depends_on: | ||
- db | ||
- rabbitmq | ||
``` | ||
#### Running a local version of the django app docker | ||
The green web platform is designed so that the different parts can be run on different servers, in various configurations. | ||
To run just the django application, once the container is built, you can run it like so: | ||
``` | ||
docker run --env-file .env.prod.docker -it greenweb-app bash | ||
``` | ||
|
||
This will log you into the running container, where you can run `gunicorn` to serve web requests, and put greenchecks onto a message queue for looking up: | ||
|
||
``` | ||
gunicorn --bind 0.0.0.0:9000 --config gunicorn.conf.py greenweb.wsgi | ||
``` | ||
|
||
|
||
Or the `dramatiq` workers that take domains to off the message queue, look them up, and writ the results to lookup table: | ||
|
||
```shell | ||
# run dramatiq, with one thread and one process, listening on all queues | ||
python ./manage.py rundramatiq --threads 1 --process 1 | ||
``` | ||
|
||
### Making new images | ||
|
||
If you are using docker you will at some point need to make new images. | ||
|
||
Running long commands in the terminal gets tedious, so the `build_containers` script for annotated notes for a script to building a docker image. | ||
|
||
There is a Makefile task intended for automating the process of creating builds - see `docker.build` | ||
|
||
|
||
|
||
### Publish an image | ||
|
||
Once you have an image built, to use it outside a development environment it needs to be accessible over the internet. At a high level the steps are: | ||
|
||
1. build the image | ||
1. tag the image | ||
1. push the image to a registry | ||
|
||
There is a Makefile task set up for this - see `docker.release` - this tags and pushes a built image to a docker repository hosted by Scaleway. You will need to be authenticated first. | ||
|
||
```shell | ||
docker login <SCALEWAY_DOCKER_REGISTRY>/<GREENWEB_NAMESPACE> -u nologin | ||
``` | ||
|
||
You will be prompted for a password - if you don't have access to these credentials, please contact one of the green web staff. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
#!/bin/bash | ||
|
||
# this is convenience script when iterating on container builds | ||
# most of the time we would use the makefile docker.build, but | ||
# this is intended for debugging, and logging into containers | ||
# when inspecting their contents | ||
|
||
set -euo pipefail | ||
|
||
# create our container | ||
docker build . --tag greenweb-app | ||
|
||
# uncommment to log into our newly created container | ||
# docker run --env-file .env.docker --interactive --tty greenweb-app bash |
Oops, something went wrong.