[Deployment] Add submission clearing for EC2 (#4183)

* [Deployment] Add submission clearing for EC2

* Update delete_old_submissions.py

* Fix flake8 issues

scripts/deployment/delete_old_submissions.py

@@ -0,0 +1,23 @@
+import os
+import shutil
+import time
+
+# Define the directory path
+tmp_directory = '/tmp'
+
+# Calculate the cutoff time (60 seconds ago)
+cutoff_time = time.time() - 60  # 60 seconds
+
+# List all directories in /tmp starting with 'tmp'
+tmp_folders = [d for d in os.listdir(tmp_directory) if os.path.isdir(os.path.join(tmp_directory, d)) and d.startswith('tmp')]
+
+for folder in tmp_folders:
+    folder_path = os.path.join(tmp_directory, folder)
+
+    # Check if the folder was created more than 60 seconds ago
+    folder_creation_time = os.path.getctime(folder_path)
+    if folder_creation_time < cutoff_time:
+        shutil.rmtree(folder_path)
+        print(f"Deleted folder: {folder_path}")
+    else:
+        print(f"Skipped folder: {folder_path} (created less than 60 seconds ago)")

scripts/deployment/deploy_ec2_worker.sh

@@ -1,32 +1,32 @@
 #!/bin/bash
 
 # Step 1: Updating package repository
-echo "Step 1/10: Updating package repository"
+echo "Step 1/11: Updating package repository"
 sudo apt-get update
 
 # Step 2: Cloning EvalAI repository at /home/ubuntu
-echo "Step 2/10: Cloning EvalAI repository"
+echo "Step 2/11: Cloning EvalAI repository"
 cd /home/ubuntu
 git clone https://github.com/Cloud-CV/EvalAI.git
 cd EvalAI
 
 # Step 3: Installing awscli
-echo "Step 3/10: Installing awscli"
+echo "Step 3/11: Installing awscli"
 sudo apt install awscli -y
 
 # Step 4: Installing docker-compose
-echo "Step 4/10: Installing docker-compose"
+echo "Step 4/11: Installing docker-compose"
 sudo apt install docker-compose -y
 sudo groupadd docker
 
 # Step 5: Adding user to docker group
-echo "Step 5/10: Adding user to docker group"
+echo "Step 5/11: Adding user to docker group"
 sudo apt -V install gnupg2 pass -y
 sudo usermod -aG docker $USER
 newgrp docker
 
 # Step 6: Configuring AWS credentials
-echo "Step 6/10: Configuring AWS credentials"
+echo "Step 6/11: Configuring AWS credentials"
 aws configure set aws_access_key_id ${AWS_ACCESS_KEY_ID}
 aws configure set aws_secret_access_key ${AWS_SECRET_ACCESS_KEY}
 aws configure set default.region ${AWS_REGION}
@@ -37,22 +37,23 @@ export TRAVIS_BRANCH=${ENVIRONMENT}
 eval $(aws ecr get-login --no-include-email)
 
 # Step 7: Copying Docker environment file
-echo "Step 7/10: Copying Docker environment file"
+echo "Step 7/11: Copying Docker environment file"
 aws s3 cp s3://cloudcv-secrets/evalai/${ENVIRONMENT}/docker_${ENVIRONMENT}.env ./docker/prod/docker_${ENVIRONMENT}.env
 
 if [ "${CUSTOM_WORKER_IMAGE}" = "" ]; 
 then
     # Step 8: Pulling worker Docker image
-    echo "Step 8/10: Pulling worker Docker image"
+    echo "Step 8/11: Pulling worker Docker image"
     docker-compose -f docker-compose-${ENVIRONMENT}.yml pull worker
 else
     # if using custom image from worker_image_url
+    echo "Step 8/11: Pulling worker Docker image"
     echo "Using custom worker image: ${CUSTOM_WORKER_IMAGE}"
     docker pull ${CUSTOM_WORKER_IMAGE}
 fi
 
 # Step 9: Running worker Docker container
-echo "Step 9/10: Running worker Docker container"
+echo "Step 9/11: Running worker Docker container"
 if [ "${CUSTOM_WORKER_IMAGE}" = "" ]; 
 then
     # If using default image from Step 8
@@ -62,9 +63,14 @@ else
     docker run --name=worker_${QUEUE} -e CHALLENGE_QUEUE=${QUEUE} -e CHALLENGE_PK=${PK} -d ${CUSTOM_WORKER_IMAGE}
 fi
 
-# Step 10: Setting up crontab
-echo "Step 10/10: Setting up crontab"
+# Step 10: Add submission clearing script
+echo "Step 10/11: Add submission clearing script to docker container"
+docker cp "scripts/deployment/delete_old_submissions.py" "worker_${QUEUE}:/code/delete_old_submissions.py"
+
+# Step 11: Setting up crontab
+echo "Step 11/11: Setting up crontab"
 echo "@reboot docker restart worker_${QUEUE}" >> workercron
+echo "@reboot docker exec -it worker_${QUEUE} python delete_old_submissions.py" >> workercron
 crontab workercron
 rm workercron