feat 1133 files transfer to ACF Titan

tdrs-backend/Pipfile

@@ -45,6 +45,11 @@ pyjwt = "==2.4.0"
 requests = "==2.27.1"
 wait-for-it = "==2.2.0"
 requests-mock = "==1.9.3"
+celery = "*"
+redis = "*"
+flower = "1.1.0"
+django-celery-beat = "2.3.0"
+paramiko = "2.11.0"
 
 [requires]
 python_version = "3.10.4"

tdrs-backend/docker-compose.local.yml

@@ -22,6 +22,8 @@ services:
       - DATA_DIR=/tmp/localstack/data
       - AWS_BUCKET=tdp-datafiles-localstack
       - AWS_REGION_NAME=us-gov-west-1
+      - HOSTNAME=localstack
+      - HOSTNAME_EXTERNAL=localstack
     ports:
       - "4566:4566"
     volumes:

tdrs-backend/docker-compose.yml

@@ -67,19 +67,30 @@ services:
       - AMS_CLIENT_ID
       - AMS_CLIENT_SECRET
       - AMS_CONFIGURATION_ENDPOINT
+      - ACFTITAN_HOST
+      - ACFTITAN_KEY
+      - ACFTITAN_USERNAME
     volumes:
       - .:/tdpapp
     image: tdp
     build: .
     command: >
       bash -c "./wait_for_services.sh &&
-               ./gunicorn_start.sh"
+               ./gunicorn_start.sh && celery -A tdpservice.settings worker -l info"
     ports:
       - "8080:8080"
+      - "5555:5555"
     depends_on:
       - clamav-rest
       - localstack
       - postgres
+      - redis-server
+
+  redis-server:
+    image: "redis:alpine"
+    command: redis-server
+    ports:
+      - "6379:6379"
 
 volumes:
   localstack_data:

tdrs-backend/gunicorn_start.sh

@@ -6,6 +6,13 @@ python manage.py makemigrations
 python manage.py migrate
 python manage.py populate_stts
 python manage.py collectstatic --noinput
+
+#
+celery -A tdpservice.settings worker -l info &
+sleep 5
+celery -A tdpservice.settings --broker=redis://redis-server:6379 flower &
+celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler &
+
 echo "Starting Gunicorn"
 if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then
     gunicorn_params="--bind 0.0.0.0:8080 --timeout 10 --workers 3 --reload --log-level $LOGGING_LEVEL"
@@ -15,4 +22,4 @@ fi
 
 gunicorn_cmd="gunicorn tdpservice.wsgi:application $gunicorn_params"
 
-exec $gunicorn_cmd
+exec $gunicorn_cmd

tdrs-backend/tasks/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

tdrs-backend/tasks/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

tdrs-backend/tasks/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

tdrs-backend/tdpservice/data_files/test/test_models.py

@@ -3,7 +3,7 @@
 
 from tdpservice.stts.models import STT
 
-from ..models import DataFile
+from tdpservice.data_files.models import DataFile
 
 
 @pytest.mark.django_db

tdrs-backend/tdpservice/data_files/views.py

@@ -12,6 +12,7 @@
 from rest_framework.viewsets import ModelViewSet
 from rest_framework.decorators import action
 from wsgiref.util import FileWrapper
+from rest_framework import status
 
 from tdpservice.data_files.serializers import DataFileSerializer
 from tdpservice.data_files.models import DataFile
@@ -33,6 +34,8 @@ class Meta:
         fields = ['stt', 'quarter', 'year']
 
 
+from tdpservice.scheduling import tasks
+
 class DataFileViewSet(ModelViewSet):
     """Data file views."""
 
@@ -51,6 +54,16 @@ class DataFileViewSet(ModelViewSet):
     # we will be able to appropriately refer to the latest versions only.
     ordering = ['-version']
 
+    def create(self, request, *args, **kwargs):
+        response = super().create(request, *args, **kwargs)
+
+        # Upload to ACF-TITAN only if file is passed the virus scan and created
+        if response.status_code == status.HTTP_201_CREATED:
+            tasks.upload.delay(
+                data_file_pk=response.data.get('id')
+            )
+        return response
+
     def filter_queryset(self, queryset):
         """Only apply filters to the list action."""
         if self.action != 'list':

tdrs-backend/tdpservice/scheduling/__init__.py

@@ -0,0 +1,28 @@
+from django.conf import settings
+settings.LOCAL_KEY = """-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEAun8bpLADVmGiq9BkGBmcPP79L7vPEeQBfC50YosrKh04i0Djvmrl
+2UPG3TJitk4ngWOcH+dz9uEVGDLVauDBQ5/m9x83q3+rP+RWgCUJq0sQ65t5zpuTJniplx
+LpalJTWOIj2yIPnZqqxSgRlXZX2uXhSfPDjkXfz0w6uxIzJrG+ak573G/bN+kWFjakI4Zo
+MzBZ95qq8Cr3EFkLhbNZzRFmUpHMApWaL8A5cqcSGCuS6q6Hozs+AW/kjPAgwH+6zs14Va
+aZTPUwaPWYW+GWn6Fwknex0iJAd2Nm6PwBod3mn4+ogdl+gFDB2qmwI+JVCg3qrap7xQdP
+fnxjpXFC7QAAA9gzkzJFM5MyRQAAAAdzc2gtcnNhAAABAQC6fxuksANWYaKr0GQYGZw8/v
+0vu88R5AF8LnRiiysqHTiLQOO+auXZQ8bdMmK2TieBY5wf53P24RUYMtVq4MFDn+b3Hzer
+f6s/5FaAJQmrSxDrm3nOm5MmeKmXEulqUlNY4iPbIg+dmqrFKBGVdlfa5eFJ88OORd/PTD
+q7EjMmsb5qTnvcb9s36RYWNqQjhmgzMFn3mqrwKvcQWQuFs1nNEWZSkcwClZovwDlypxIY
+K5LqroejOz4Bb+SM8CDAf7rOzXhVpplM9TBo9Zhb4ZafoXCSd7HSIkB3Y2bo/AGh3eafj6
+iB2X6AUMHaqbAj4lUKDeqtqnvFB09+fGOlcULtAAAAAwEAAQAAAQBX18kYgEOCK4jorsaG
+x4FdBaoSEr9NKP0+Fc+CHxUcyBi7FrxvE4DiHleMFtCCGDvpaRa/vFBV59z8wezeY7EEfk
+vW+KBLKl5qQM4xm8B7K2dz4o60L3lXNJyE6UaCavaRNis3a0q5rIFlTiNCKDTniUHdiHED
+EYyC4uAC/2aSgXZdW/+uythxLVsRmpA3weFrMAfMRCM45Luxj2N8LZ4R213aPikX+dSMHK
+hxnxvGL9zDwMRR3v4jqBYqXaLPpZKedyvrcNc3TqG+RhUsuSU/0j5YEk683zTxWmIc4Ze6
+j/PDXFrSRREwZCuzeL4emkDUSUKdh0FJ1vCEDhT+DuptAAAAgAdM+SFIjOjMd4L+7GidVF
+VE3f+OL5Z6rPRNmwx4ifibukHdk9wsHp55yfccxr013xvM/EOmnhNqb5uNj/MgN981/ZFb
+RJobRG4NuH1HHZMd9F8BE5JtkUiuuhryxEzfwTpsgEJuCtR5BW8v9laKYkaBViejZ/XM2M
+bjIKCeOooOAAAAgQDarA/9tVL2YR0VupZfQQdZL+4lO2RtvddjdSKEECvckxSRIKVA/9HY
+KJRHwrNIzA8sPuxNBR+9BRp+cw3cgTIQ/Rp+ISF9PFVxTMnM//vcsUTUxziYvaal3DDIpb
+f8KcPJ6ECSVq/TYPYQ+wCLoPM49c8TnW8PxtDb3f+fI5WBdwAAAIEA2lT6BkcR9A6FHB+j
+Ad2PpLYJqef1ZCQJrhvQRlQfrqhmAie7TdTWi7eH/jUksh+4j4rlobQ22ryTBKMA91npq8
+8lhuJnULmWzl7CBTR8YQ/6WbY58PTadkk79e9w/4WOWp8b0SxMwtZDnESs5x1dfuUyqqfS
+rFBHiFOY1MK2F7sAAAAdbXNvaGFuaUBNb3MtTWFjQm9vay1Qcm8ubG9jYWwBAgMEBQY=
+-----END OPENSSH PRIVATE KEY-----"""

tdrs-backend/tdpservice/scheduling/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

tdrs-backend/tdpservice/scheduling/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class TasksConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'tdpservice.scheduling'

tdrs-backend/tdpservice/scheduling/tasks.py

@@ -0,0 +1,65 @@
+from __future__ import absolute_import
+
+import os
+
+from celery import shared_task
+from django.conf import settings
+import datetime
+import paramiko
+import logging
+from tdpservice.data_files.models import DataFile
+logger = logging.getLogger(__name__)
+
+server_address = settings.SERVER_ADDRESS
+local_key = settings.LOCAL_KEY
+username = settings.USERNAME
+
+
+def write_key_to_file(private_key):
+    """
+    Paramiko require the key in file object format
+    """
+    with open('temp_key_file', 'w') as f:
+        f.write(private_key)
+        f.close()
+    return 'temp_key_file'
+
+
+@shared_task
+def upload(data_file_pk):
+    """
+    This task uploads the file in DataFile object with
+    pk = data_file_pk
+    to sftp server as defined in Settings file
+    """
+    data_file = DataFile.objects.get(id=data_file_pk)
+    destination = str(data_file.file).split('/')[-1]
+    today_date = datetime.datetime.today()
+    upper_directory_name = today_date.strftime('%Y%m%d')
+    lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter))
+
+    transport = paramiko.SSHClient()
+    transport.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    transport.connect(server_address, key_filename=write_key_to_file(local_key), username=username)
+    sftp = transport.open_sftp()
+
+    transport.exec_command('mkdir -p ' + upper_directory_name +
+                           '/' + lower_directory_name)
+
+    f = data_file.file.read()
+    with open(destination, 'wb') as f1:
+        f1.write(f)
+        f1.close()
+    logger.info(os.listdir())
+
+    # Upload file
+    try:
+        # sftp = transport.open_sftp()
+        # temp file can be the file object
+        sftp.put(destination, upper_directory_name + '/' + lower_directory_name + '/' + destination)
+        os.remove(destination)
+        logger.info('File {} has been successfully uploaded to {}'.format(destination, server_address))
+        return True
+    except Exception as e:
+        logger.error('Failed to upload {} with error:{}'.format(destination, e))
+        return False

tdrs-backend/tdpservice/scheduling/test/test_file_upload.py

@@ -0,0 +1,30 @@
+"""Integration test(s) for clamav-rest operations."""
+from os import remove
+from requests.sessions import Session
+import pytest
+
+from django.contrib.admin import site as admin_site
+from django.core.files.base import File
+from rest_framework.status import HTTP_400_BAD_REQUEST
+from tdpservice.scheduling.tasks import upload
+
+"""
+WRITE TEST:
+
+1. CREATE FILE -> clean_file fixture -> DONE
+2. UPLOAD TO ACF_TITAN -> 
+3. SSH TO ACF TITAN AND MAKE SURE THE FILE EXISTS
+"""
+
+
+from tdpservice.data_files.test.factories import DataFileFactory
+
+
+@pytest.fixture()
+def upload_file():
+    return DataFileFactory()
+
+
+@pytest.mark.django_db
+def test_upload_server(upload_file):
+    assert upload_file.original_filename == 'data_file.txt'

tdrs-backend/tdpservice/security/__init__.py

@@ -0,0 +1 @@
+

tdrs-backend/tdpservice/settings/__init__.py

@@ -0,0 +1,9 @@
+from __future__ import absolute_import
+import logging
+import os
+from django.conf import settings
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+__all__ = ['celery_app']

tdrs-backend/tdpservice/settings/celery.py

@@ -0,0 +1,36 @@
+from __future__ import absolute_import
+import os
+from celery import Celery, shared_task
+from django.conf import settings
+import configurations
+from celery.schedules import crontab
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.local")
+os.environ.setdefault("DJANGO_CONFIGURATION", "Local")
+
+configurations.setup()
+
+app = Celery('settings')
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))
+
+
+app.conf.beat_schedule = {
+    'example': {
+        'task': 'tdpservice.scheduling.tasks.run_backup',
+        'schedule': crontab(minute='10'),
+        'args': ['lolo'],
+    },
+}

tdrs-backend/tdpservice/settings/common.py

@@ -12,7 +12,6 @@
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-
 def get_required_env_var_setting(
     env_var_name: str,
     setting_name: Optional[str] = None
@@ -48,13 +47,15 @@ class Common(Configuration):
         "corsheaders",
         "django_extensions",
         "drf_yasg",
+        "django_celery_beat",
         "storages",
         # Local apps
         "tdpservice.core.apps.CoreConfig",
         "tdpservice.users",
         "tdpservice.stts",
         "tdpservice.data_files",
         "tdpservice.security",
+        "tdpservice.scheduling"
     )
 
     # https://docs.djangoproject.com/en/2.0/topics/http/middleware/
@@ -69,7 +70,7 @@ class Common(Configuration):
         "corsheaders.middleware.CorsMiddleware",
         "tdpservice.users.api.middleware.AuthUpdateMiddleware",
         "csp.middleware.CSPMiddleware",
-        "tdpservice.middleware.NoCacheMiddleware",
+        "tdpservice.middleware.NoCacheMiddleware"
     )
 
     APP_NAME = "dev"
@@ -397,3 +398,16 @@ class Common(Configuration):
         'AMS_CLIENT_SECRET',
         ''
     )
+
+    # sftp config
+    SERVER_ADDRESS = os.getenv('ACFTITAN_HOST', '')
+    LOCAL_KEY = os.getenv('ACFTITAN_KEY', '')
+    USERNAME = os.getenv('ACFTITAN_USERNAME', '')
+
+    # CELERY CONFIG
+    CELERY_BROKER_URL = "redis://redis-server:6379"
+    CELERY_RESULT_BACKEND = "redis://redis-server:6379"
+    CELERY_ACCEPT_CONTENT = ['application/json']
+    CELERY_TASK_SERIALIZER = 'json'
+    CELERY_RESULT_SERIALIZER = 'json'
+    CELERY_TIMEZONE = 'UTC'

tdrs-backend/tdpservice/stts/test/test_api.py

@@ -3,7 +3,7 @@
 from django.contrib.auth import get_user_model
 from django.urls import reverse
 from rest_framework import status
-from ...stts.models import STT, Region
+from tdpservice.stts.models import STT, Region
 
 User = get_user_model()
 

tdrs-backend/tdpservice/stts/test/test_commands.py

@@ -2,7 +2,7 @@
 
 from django.core.management import call_command
 import pytest
-from ..models import Region, STT
+from tdpservice.stts.models import Region, STT
 
 
 @pytest.mark.django_db

tdrs-backend/tdpservice/stts/test/test_models.py

@@ -1,7 +1,7 @@
 """Module for testing the stt and region models."""
 import pytest
 
-from ..models import STT, Region
+from tdpservice.stts.models import STT, Region
 
 
 @pytest.mark.django_db