From 5cedd72048a6a742689d09920e20dd7faf966529 Mon Sep 17 00:00:00 2001 From: Gunjan Chhablani Date: Mon, 2 Oct 2023 18:57:29 -0400 Subject: [PATCH] [Deployment] Add submission clearing for EC2 (#4183) * [Deployment] Add submission clearing for EC2 * Update delete_old_submissions.py * Fix flake8 issues --- scripts/deployment/delete_old_submissions.py | 23 ++++++++++++++++ scripts/deployment/deploy_ec2_worker.sh | 28 ++++++++++++-------- 2 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 scripts/deployment/delete_old_submissions.py diff --git a/scripts/deployment/delete_old_submissions.py b/scripts/deployment/delete_old_submissions.py new file mode 100644 index 0000000000..0de18ba13c --- /dev/null +++ b/scripts/deployment/delete_old_submissions.py @@ -0,0 +1,23 @@ +import os +import shutil +import time + +# Define the directory path +tmp_directory = '/tmp' + +# Calculate the cutoff time (60 seconds ago) +cutoff_time = time.time() - 60 # 60 seconds + +# List all directories in /tmp starting with 'tmp' +tmp_folders = [d for d in os.listdir(tmp_directory) if os.path.isdir(os.path.join(tmp_directory, d)) and d.startswith('tmp')] + +for folder in tmp_folders: + folder_path = os.path.join(tmp_directory, folder) + + # Check if the folder was created more than 60 seconds ago + folder_creation_time = os.path.getctime(folder_path) + if folder_creation_time < cutoff_time: + shutil.rmtree(folder_path) + print(f"Deleted folder: {folder_path}") + else: + print(f"Skipped folder: {folder_path} (created less than 60 seconds ago)") diff --git a/scripts/deployment/deploy_ec2_worker.sh b/scripts/deployment/deploy_ec2_worker.sh index 90068e561b..ec5e97bd45 100644 --- a/scripts/deployment/deploy_ec2_worker.sh +++ b/scripts/deployment/deploy_ec2_worker.sh @@ -1,32 +1,32 @@ #!/bin/bash # Step 1: Updating package repository -echo "Step 1/10: Updating package repository" +echo "Step 1/11: Updating package repository" sudo apt-get update # Step 2: Cloning EvalAI repository at /home/ubuntu -echo "Step 2/10: Cloning EvalAI repository" +echo "Step 2/11: Cloning EvalAI repository" cd /home/ubuntu git clone https://github.com/Cloud-CV/EvalAI.git cd EvalAI # Step 3: Installing awscli -echo "Step 3/10: Installing awscli" +echo "Step 3/11: Installing awscli" sudo apt install awscli -y # Step 4: Installing docker-compose -echo "Step 4/10: Installing docker-compose" +echo "Step 4/11: Installing docker-compose" sudo apt install docker-compose -y sudo groupadd docker # Step 5: Adding user to docker group -echo "Step 5/10: Adding user to docker group" +echo "Step 5/11: Adding user to docker group" sudo apt -V install gnupg2 pass -y sudo usermod -aG docker $USER newgrp docker # Step 6: Configuring AWS credentials -echo "Step 6/10: Configuring AWS credentials" +echo "Step 6/11: Configuring AWS credentials" aws configure set aws_access_key_id ${AWS_ACCESS_KEY_ID} aws configure set aws_secret_access_key ${AWS_SECRET_ACCESS_KEY} aws configure set default.region ${AWS_REGION} @@ -37,22 +37,23 @@ export TRAVIS_BRANCH=${ENVIRONMENT} eval $(aws ecr get-login --no-include-email) # Step 7: Copying Docker environment file -echo "Step 7/10: Copying Docker environment file" +echo "Step 7/11: Copying Docker environment file" aws s3 cp s3://cloudcv-secrets/evalai/${ENVIRONMENT}/docker_${ENVIRONMENT}.env ./docker/prod/docker_${ENVIRONMENT}.env if [ "${CUSTOM_WORKER_IMAGE}" = "" ]; then # Step 8: Pulling worker Docker image - echo "Step 8/10: Pulling worker Docker image" + echo "Step 8/11: Pulling worker Docker image" docker-compose -f docker-compose-${ENVIRONMENT}.yml pull worker else # if using custom image from worker_image_url + echo "Step 8/11: Pulling worker Docker image" echo "Using custom worker image: ${CUSTOM_WORKER_IMAGE}" docker pull ${CUSTOM_WORKER_IMAGE} fi # Step 9: Running worker Docker container -echo "Step 9/10: Running worker Docker container" +echo "Step 9/11: Running worker Docker container" if [ "${CUSTOM_WORKER_IMAGE}" = "" ]; then # If using default image from Step 8 @@ -62,9 +63,14 @@ else docker run --name=worker_${QUEUE} -e CHALLENGE_QUEUE=${QUEUE} -e CHALLENGE_PK=${PK} -d ${CUSTOM_WORKER_IMAGE} fi -# Step 10: Setting up crontab -echo "Step 10/10: Setting up crontab" +# Step 10: Add submission clearing script +echo "Step 10/11: Add submission clearing script to docker container" +docker cp "scripts/deployment/delete_old_submissions.py" "worker_${QUEUE}:/code/delete_old_submissions.py" + +# Step 11: Setting up crontab +echo "Step 11/11: Setting up crontab" echo "@reboot docker restart worker_${QUEUE}" >> workercron +echo "@reboot docker exec -it worker_${QUEUE} python delete_old_submissions.py" >> workercron crontab workercron rm workercron