1755 - As a data analyst, I need to be able to upload SSP data files

tdrs-backend/docker-compose.local.yml

@@ -58,7 +58,7 @@ services:
       - ACFTITAN_HOST
       - ACFTITAN_KEY
       - ACFTITAN_USERNAME
-      - REDIS_URI
+      - REDIS_URI=redis://redis-server:6379
       - REDIS_SERVER_LOCAL=TRUE
       - ACFTITAN_SFTP_PYTEST
     volumes:

tdrs-backend/docker-compose.yml

@@ -70,7 +70,7 @@ services:
       - ACFTITAN_HOST
       - ACFTITAN_KEY
       - ACFTITAN_USERNAME
-      - REDIS_URI
+      - REDIS_URI=redis://redis-server:6379
       - REDIS_SERVER_LOCAL=TRUE
       - ACFTITAN_SFTP_PYTEST
     volumes:

tdrs-backend/manifest.buildpack.yml

@@ -4,6 +4,8 @@ applications:
   memory: 512M
   instances: 1
   disk_quota: 2G
+  env:
+    REDIS_URI: redis://localhost:6379
   buildpacks:
   - https://github.com/cloudfoundry/apt-buildpack
   - https://github.com/cloudfoundry/python-buildpack.git#v1.7.55

tdrs-backend/tdpservice/conftest.py

@@ -218,7 +218,8 @@ def base_data_file_data(fake_file_name, data_analyst):
         "user": str(data_analyst.id),
         "quarter": "Q1",
         "year": 2020,
-        "stt": int(data_analyst.stt.id)
+        "stt": int(data_analyst.stt.id),
+        "ssp": False,
     }
 
 

tdrs-backend/tdpservice/data_files/models.py

@@ -149,9 +149,14 @@ class Meta:
     @property
     def filename(self):
         """Return the correct filename for this data file."""
+        # This logic is temporary until the section choices for tribe and SSP are available
         if str(self.stt.type).lower() == 'tribe':
             return self.stt.filenames.get(
-                ('Tribal ' if 'Tribal' not in self.section else '') + self.section,
+                ('Tribal ' if 'Tribal' not in self.section else '') + self.section,  # This is temporary logic
+                None)
+        elif self.stt.ssp:  # Currently, this doesn't exists and has to be added
+            return self.stt.filenames.get(
+                ('SSP ' if 'SSP' not in self.section else '') + self.section,  # This is temporary logic
                 None)
         else:
             return self.stt.filenames.get(self.section, None)

tdrs-backend/tdpservice/data_files/serializers.py

@@ -19,6 +19,7 @@ class DataFileSerializer(serializers.ModelSerializer):
     file = serializers.FileField(write_only=True)
     stt = serializers.PrimaryKeyRelatedField(queryset=STT.objects.all())
     user = serializers.PrimaryKeyRelatedField(queryset=User.objects.all())
+    ssp = serializers.BooleanField(write_only=True)
 
     class Meta:
         """Metadata."""
@@ -35,13 +36,16 @@ class Meta:
             "year",
             "quarter",
             "section",
-            "created_at"
+            "created_at",
+            "ssp",
         ]
 
     def create(self, validated_data):
         """Create a new entry with a new version number."""
+        ssp = validated_data.pop('ssp')
+        if ssp:
+            validated_data['section'] = 'SSP ' + validated_data['section']
         data_file = DataFile.create_new_version(validated_data)
-
         # Determine the matching ClamAVFileScan for this DataFile.
         av_scan = ClamAVFileScan.objects.filter(
             file_name=data_file.original_filename,

tdrs-backend/tdpservice/data_files/test/test_api.py

@@ -206,6 +206,24 @@ def test_download_data_file_file_rejected_for_other_stt(
 
         assert response.status_code == status.HTTP_403_FORBIDDEN
 
+    def test_data_files_data_upload_ssp(
+        self, api_client, data_file_data,
+    ):
+        """Test that when Data Analysts upload file with ssp true the section name is updated."""
+        data_file_data['ssp'] = True
+
+        response = self.post_data_file_file(api_client, data_file_data)
+        assert response.data['section'] == 'SSP Active Case Data'
+
+    def test_data_files_data_upload_tanf(
+        self, api_client, data_file_data,
+    ):
+        """Test that when Data Analysts upload file with ssp true the section name is updated."""
+        data_file_data['ssp'] = False
+
+        response = self.post_data_file_file(api_client, data_file_data)
+        assert response.data['section'] == 'Active Case Data'
+
     def test_data_analyst_gets_email_when_user_uploads_report_for_their_stt(
         self, api_client, data_file_data, user
     ):

tdrs-backend/tdpservice/data_files/views.py

@@ -97,7 +97,17 @@ def filter_queryset(self, queryset):
         """Only apply filters to the list action."""
         if self.action != 'list':
             self.filterset_class = None
-        return super().filter_queryset(queryset)
+
+        print('hey')
+        print('is anybody out there')
+        print(self.request.query_params)
+        is_ssp_file_type = True if self.request.query_params.get('file_type') == 'ssp-moe' else False
+        if is_ssp_file_type:
+            filtered_queryset = queryset.filter(section__contains='SSP')
+        else:
+            filtered_queryset = queryset.exclude(section__contains='SSP')
+
+        return super().filter_queryset(filtered_queryset)
 
     def get_serializer_context(self):
         """Retrieve additional context required by serializer."""

tdrs-backend/tdpservice/stts/management/commands/populate_stts.py

@@ -21,54 +21,31 @@ def _populate_regions():
             Region.objects.get_or_create(id=row["Id"])
         Region.objects.get_or_create(id=1000)
 
-
-def _get_states():
-    with open(DATA_DIR / "states.csv") as csvfile:
+def _load_csv(filename, entity):
+    with open(DATA_DIR / filename) as csvfile:
         reader = csv.DictReader(csvfile)
-        return [
-            STT(
-                code=row["Code"],
-                name=row["Name"],
-                region_id=row["Region"],
-                type=STT.EntityType.STATE,
-                filenames=json.loads(row["filenames"].replace('\'', '"')),
-                stt_code=row["STT_CODE"],
-            )
-            for row in reader
-        ]
-
-
-def _get_territories():
-    with open(DATA_DIR / "territories.csv") as csvfile:
-        reader = csv.DictReader(csvfile)
-        return [
-            STT(
-                code=row["Code"],
-                name=row["Name"],
-                region_id=row["Region"],
-                type=STT.EntityType.TERRITORY,
-                filenames=json.loads(row["filenames"].replace('\'', '"')),
-                stt_code=row["STT_CODE"],
-            )
-            for row in reader
-        ]
-
 
-def _populate_tribes():
-    with open(DATA_DIR / "tribes.csv") as csvfile:
-        reader = csv.DictReader(csvfile)
-        stts = [
-            STT(
-                name=row["Name"],
-                region_id=row["Region"],
-                state=STT.objects.get(code=row["Code"]),
-                type=STT.EntityType.TRIBE,
-                filenames=json.loads(row["filenames"].replace('\'', '"')),
-                stt_code=row["STT_CODE"],
-            )
-            for row in reader
-        ]
-        STT.objects.bulk_create(stts, ignore_conflicts=True)
+        for row in reader:
+            stt, stt_created = STT.objects.get_or_create(name=row["Name"])
+            if stt_created:  # These lines are spammy, should remove before merge
+                logger.debug("Created new entry for " + row["Name"])
+            else:
+                logger.debug("Found STT " + row["Name"] + ", will sync with data csv.")
+
+            stt.code = row["Code"]
+            stt.region_id = row["Region"]
+            if filename == "tribes.csv":
+                stt.state = STT.objects.get(code=row["Code"], type=STT.EntityType.STATE)
+
+            stt.type = entity
+            stt.filenames = json.loads(row["filenames"].replace('\'', '"'))
+            stt.stt_code = row["STT_CODE"]
+            stt.ssp = row["SSP"]
+            # TODO: Was seeing lots of references to STT.objects.filter(pk=...
+            #       We could probably one-line this but we'd miss .save() signals
+            #       https://stackoverflow.com/questions/41744096/
+            # TODO: we should finish the last columns from the csvs: Sample, SSN_Encrypted
+            stt.save()
 
 
 class Command(BaseCommand):
@@ -79,8 +56,14 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         """Populate the various regions, states, territories, and tribes."""
         _populate_regions()
-        stts = _get_states()
-        stts.extend(_get_territories())
-        STT.objects.bulk_create(stts, ignore_conflicts=True)
-        _populate_tribes()
+
+        stt_map = [
+            ("states.csv", STT.EntityType.STATE),
+            ("territories.csv", STT.EntityType.TERRITORY),
+            ("tribes.csv", STT.EntityType.TRIBE)
+        ]
+
+        for datafile, entity in stt_map:
+            _load_csv(datafile, entity)
+
         logger.info("STT import executed by Admin at %s", timezone.now())

tdrs-backend/tdpservice/stts/migrations/0006_alter_stt_filenames.py

@@ -1,49 +1,9 @@
 # Generated by Django 3.2.13 on 2022-06-08 14:43
 
-import csv
-import json
-from pathlib import Path
-
 from django.core.management import call_command
 from django.db import migrations, models
 
 
-def add_filenames(apps, schema_editor):
-    call_command("populate_stts")
-    data_dir = Path(__file__).resolve().parent.parent  /"management" / "commands" / "data"
-    STT = apps.get_model('stts','STT')
-
-    fieldnames = ["Code", "Name", "Region", "STT_CODE", "Sample", "SSP", "SSN_Encrypted", "filenames"]
-    with open(data_dir / "states.csv", "r") as csvfile:
-        reader = csv.DictReader(csvfile)
-        rows=[]
-        for row in reader:
-            filenames = row["filenames"] = row["filenames"].replace('\'', '"')
-            rows.append(row)
-            state = STT.objects.get(name=row["Name"])
-            state.filenames = json.loads(filenames)
-            state.save()
-
-    with open(data_dir / "tribes.csv", "r") as csvfile:
-        reader = csv.DictReader(csvfile)
-        rows=[]
-        for row in reader:
-            filenames = row["filenames"] = row["filenames"].replace('\'', '"')
-            rows.append(row)
-            tribe = STT.objects.get(name=row["Name"])
-            tribe.filenames = json.loads(filenames)
-            tribe.save()
-
-    with open(data_dir / "territories.csv", "r") as csvfile:
-        reader = csv.DictReader(csvfile)
-        rows=[]
-        for row in reader:
-            filenames = row["filenames"] = row["filenames"].replace('\'', '"')
-            rows.append(row)
-            territory = STT.objects.get(name=row["Name"])
-            territory.filenames = json.loads(filenames)
-            territory.save()
-
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -56,6 +16,4 @@ class Migration(migrations.Migration):
             name='filenames',
             field=models.JSONField(blank=True, max_length=512, null=True),
         ),
-        migrations.RunPython(add_filenames)
     ]
-

tdrs-backend/tdpservice/stts/migrations/0007_stt_ssp.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2022-10-28 20:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('stts', '0006_alter_stt_filenames'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='stt',
+            name='ssp',
+            field=models.BooleanField(default=False, null=True),
+        ),
+    ]

tdrs-backend/tdpservice/stts/models.py

@@ -37,6 +37,7 @@ class EntityType(models.TextChoices):
     stt_code = models.PositiveIntegerField(blank=True, null=True)
     # Tribes have a state, which we need to store.
     state = models.ForeignKey("self", on_delete=models.CASCADE, blank=True, null=True)
+    ssp = models.BooleanField(default=False, null=True)
 
     class Meta:
         """Metadata."""

tdrs-backend/tdpservice/stts/serializers.py

@@ -14,7 +14,7 @@ class Meta:
         """Metadata."""
 
         model = STT
-        fields = ["id", "type", "code", "name", "region"]
+        fields = ["id", "type", "code", "name", "region", "ssp"]
 
     def get_code(self, obj):
         """Return the state code."""

tdrs-backend/tdpservice/users/test/test_api/test_set_profile.py

@@ -38,30 +38,6 @@ def test_set_profile_data(api_client, user):
     assert user.last_name == "Bloggs"
 
 
-@pytest.mark.django_db
-def test_user_can_request_access(api_client, user, stt):
-    """Test `access_request` endpoint updates the `account_approval_status` field to `Access Request`."""
-    api_client.login(username=user.username, password="test_password")
-
-    response = api_client.patch(
-            "/v1/users/request_access/",
-            {"first_name": "Joe", "last_name": "Bloggs", "stt": stt.id, "email": user.username},
-            format="json",
-        )
-    assert response.data == {
-            "id": str(user.id),
-            "email": user.username,
-            "first_name": "Joe",
-            "last_name": "Bloggs",
-            "access_request": False,  # old value no longer touched
-            "account_approval_status": "Access request",  # new value updated
-            "stt": {"id": stt.id, "type": stt.type, "code": stt.code, "name": stt.name, "region": stt.region.id},
-            "region": None,
-            "roles": [],
-        }
-
-    # TODO: In the future, we would like to test that users can be activated and their roles are correctly assigned.
-
 @pytest.mark.django_db
 def test_cannot_set_account_approval_status_through_api(api_client, user):
     """Test that the `account_approval_status` field cannot be updated through an api call to `set_profile`."""

tdrs-backend/temp_key_file

@@ -0,0 +1,23 @@
+-----BEGIN RSA PRIVATE KEY-----MIIEpQIBAAKCAQEAsf16LYNvP6vAcxZhSueDCxIpgUbOr4Hj6f37B/nAU+EIbU+3
+i54NXTMmtDjG+xzRNOH110h3o7P3A5B7cFu+IlG0AHzlsBbfO/JAGzcLbtKnxDwn/C5gFpUNKWqvj9kgqRmrmqPLl7UjfYxZVUSRD8Gh/zIONvTl3C0/EsPn/77e0YH/wC7QDyg4rikK9Cksekp9PiskgkeqCIJTk0ETiGGhOODUBOWrhHCZBXAvbbhBe/EV
+diU0IUKU869Ci0dKKP3h6YjeDhNwrkNa7jZnXPYPRHdsmWi+RK70dsoZe/MkK2GE
+yucjsh8gCHWvltlJ7NQJ0fkrcrEdTp5K9ddVAQIDAQABAoIBAHMMDYIHt8vm/1Ek
+gSTCeiCYz70hAI3PHntr0Gv6UgelqCXH5jLXqXm5i2XoYS2FFfDhsV9DFxn4REzI
+ghFBK6fROdEq6eglEIwV7Lvqm3g5r3kXDR1i+HcARu6jZ/FJ/mNvFU2yW/Gmgtcu
+9bs4w670kPp751Y8e6sSj/dYK8hRCoQlN7B5Ald8bZpVcZA/aVmJLfGvzNBY0D+x
+wsTRvZn5Q7XG/BJL0+ZWJoSEi4SPrkENUdlHueRHpwPlzBuiGmksrpYvlnnqwvZD
+ld90hxkGgnDCmReTzBRKqSPh3wab+OKzePVrLGDFzhd4V5rSsOJl01uOHac/8QLZ
+vHCQEDECgYEA2VHvIkmwu+BpDLXAmD8aiKfzcrEGT32SDVZaNdRcrADXKbtVmSKN
+I/UCmtfwNVHZk/FMPqarnfWtqv0B62eovI8qaYNCzNUFCKW3aUcACnnMc9HKlYsf
+25jJF2X3JzMv319xYnniy77fr6RsxFw2l/7J3dg0tNeFUvPVMPt8trUCgYEA0at/
+eqzMowG/8dwFubY3t/hlwPT4HuvD1A+/yC5jThU280l7lCwRjk9Eh9024Ihy0XuP+m6af1IMzhg5fYqrF7sU7Tqxx2m+47J5iQZ+m2n6jSLDslqTcAYKj9XOHMJ3/ANe
++eZKpXxpTstfX7A5Yug9a58PjleMjdm1VpDzKJ0CgYEAhOem58FJZJ0JocxFzNZK
+0+hi6nF4+oRBHgcBhIorYsXg0JTQ9KY8yxC8VxZYwUMdXWzkxCwKKMBnRXsWAXGT
+sD2eIok0ATEFsxQl5yyUydNTRkG3M12yTgpScQza6g5T6LfmD+Oa4CALjM9x9WSv
+vqUDr7jaAv8Len/EkgA7dUECgYEAx5X5/pvZHF5BCgkIhjTXq09QBTLrsft56Tao
+t/S4YQ6+xS4w7eZZO99m+/HvGCOrMI/viVOZzBMdz12t9Dx5C1jx3bTeoFWf+X3e
+RTqicGycrZbnNLMV4DBQA4Vh82yG7KWE1luKuSbJ09CyVBMbPXSXawf5teTPDgSs
+ot/OJ90CgYEAvWYQHspz7irepznaDSwnUqbfDPNQr16IN6s3KnR/MGKl6q54uQ80
+2U/k4d3KxHMAFEUVBXSBOwFswoz0ZgR9bzMUDsoSwtnfXoyFvG1GiW20diNpjPIg
+toSg3kO4adt6TdPPc1hRlbKah+Ft20qVhvaPAmQDJD3Ow8we7UAZIas=
+-----END RSA PRIVATE KEY-----

tdrs-backend/test

@@ -0,0 +1 @@
+test