From 529380cf0a7e1387e3a06adfbc730124d40b7981 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 28 Jun 2022 09:41:02 -0700 Subject: [PATCH 001/113] added ToDo comments where the logic has to be added --- tdrs-backend/tdpservice/data_files/serializers.py | 2 ++ tdrs-backend/tdpservice/data_files/views.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index c8492e05cb..44d7467868 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -53,6 +53,8 @@ def create(self, validated_data): av_scan.data_file = data_file av_scan.save() + # TODO: Add Titan upload here + return data_file def update(self, instance, validated_data): diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index b6a2e79d71..f3c95f88e6 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -51,6 +51,11 @@ class DataFileViewSet(ModelViewSet): # we will be able to appropriately refer to the latest versions only. ordering = ['-version'] + def perform_create(self): + """ + TODO: create has to be done here + """ + def filter_queryset(self, queryset): """Only apply filters to the list action.""" if self.action != 'list': From 72527386ea05d8bb3568ae1c6b88e4586028670e Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 1 Jul 2022 06:56:46 -0700 Subject: [PATCH 002/113] 1133: redis works, celery doesn't connect --- tdrs-backend/Dockerfile | 2 + tdrs-backend/Pipfile | 2 + tdrs-backend/Pipfile.lock | 229 +++++++++++------- tdrs-backend/docker-compose.yml | 7 + .../tdpservice/scheduling/__init__.py | 5 + tdrs-backend/tdpservice/scheduling/celery.py | 23 ++ tdrs-backend/tdpservice/settings/common.py | 8 + 7 files changed, 192 insertions(+), 84 deletions(-) create mode 100644 tdrs-backend/tdpservice/scheduling/__init__.py create mode 100644 tdrs-backend/tdpservice/scheduling/celery.py diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index f43ec2a41b..1704c2c47f 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -16,6 +16,8 @@ RUN apt-get -y update RUN apt-get -y upgrade # Install a new package: RUN apt-get install -y gcc && apt-get install -y graphviz && apt-get install -y graphviz-dev +# Install Redis +# RUN apt-get install -y redis-server # Install pipenv RUN pip install --upgrade pip pipenv && pipenv install --dev --system --deploy diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index f5c487d635..cc3990c416 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -45,6 +45,8 @@ pyjwt = "==2.4.0" requests = "==2.27.1" wait-for-it = "==2.2.0" requests-mock = "==1.9.3" +celery = "*" +redis = "*" [requires] python_version = "3.10.4" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index 8a1d6eae1f..8cd5345cc2 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8985d8b501034c97d2017b78794a34f2285462c9e2950674c7383edc1dec1e06" + "sha256": "803728542591a1713c7e67f0ae6c3ef8b2057a32c5ff1c4a43d34c45654c5796" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,14 @@ ] }, "default": { + "amqp": { + "hashes": [ + "sha256:2c1b13fecc0893e946c65cbd5f36427861cffa4ea2201d8f6fca22e2a373b5e2", + "sha256:6f0956d2c23d8fa6e7691934d8c3930eadb44972cbbd1a7ae3a520f735d43359" + ], + "markers": "python_version >= '3.6'", + "version": "==5.1.1" + }, "asgiref": { "hashes": [ "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4", @@ -31,6 +39,14 @@ ], "version": "==2.0.5" }, + "async-timeout": { + "hashes": [ + "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15", + "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c" + ], + "markers": "python_version >= '3.6'", + "version": "==4.0.2" + }, "backcall": { "hashes": [ "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e", @@ -38,6 +54,13 @@ ], "version": "==0.2.0" }, + "billiard": { + "hashes": [ + "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", + "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b" + ], + "version": "==3.6.4.0" + }, "boto3": { "hashes": [ "sha256:08b6dacbe7ebe57ae8acfb7106b2728d946ae1e0c3da270caee1deb79ccbd8af", @@ -54,13 +77,21 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==1.20.112" }, + "celery": { + "hashes": [ + "sha256:138420c020cd58d6707e6257b6beda91fd39af7afde5d36c6334d175302c0e14", + "sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d" + ], + "index": "pypi", + "version": "==5.2.7" + }, "certifi": { "hashes": [ - "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", - "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], "markers": "python_version >= '3.6'", - "version": "==2022.5.18.1" + "version": "==2022.6.15" }, "cffi": { "hashes": [ @@ -133,6 +164,28 @@ "markers": "python_version >= '3.7'", "version": "==8.1.3" }, + "click-didyoumean": { + "hashes": [ + "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", + "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" + ], + "markers": "python_version < '4' and python_full_version >= '3.6.2'", + "version": "==0.3.0" + }, + "click-plugins": { + "hashes": [ + "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", + "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" + ], + "version": "==1.1.1" + }, + "click-repl": { + "hashes": [ + "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b", + "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8" + ], + "version": "==0.2.0" + }, "coreapi": { "hashes": [ "sha256:46145fcc1f7017c076a2ef684969b641d18a2991051fddec9458ad3f78ffc1cb", @@ -172,7 +225,7 @@ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186" ], - "markers": "python_version >= '3.5'", + "markers": "python_version >= '3.7'", "version": "==5.1.1" }, "deprecated": { @@ -217,10 +270,10 @@ }, "django-colorfield": { "hashes": [ - "sha256:aaebf164b7c1524c5474fdb2a4aea017bb44330842cdfabcf6a17ce20e8adf61", - "sha256:be9cad8abb4c309d89e5c3beee3cd89cb3b9042c51adcf1017cf315487df10cf" + "sha256:08d093d5b31b599b681665ab97b957045c4a4c0b93cfc434bd257378a7193c96", + "sha256:a58d1c5a56f0380439dc1305a131321b839df35ddfac2a6dbcc887de02232120" ], - "version": "==0.7.0" + "version": "==0.7.1" }, "django-configurations": { "hashes": [ @@ -357,7 +410,7 @@ "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1", "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd" ], - "markers": "python_version >= '3.8'", + "markers": "python_version >= '3.7'", "version": "==8.4.0" }, "itypes": { @@ -398,6 +451,14 @@ "index": "pypi", "version": "==1.3.1" }, + "kombu": { + "hashes": [ + "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610", + "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4" + ], + "markers": "python_version >= '3.7'", + "version": "==5.2.4" + }, "markdown": { "hashes": [ "sha256:cbb516f16218e643d8e0a95b309f77eb118cb138d39a4f27851e6a63581db874", @@ -537,11 +598,11 @@ }, "prompt-toolkit": { "hashes": [ - "sha256:62291dad495e665fca0bda814e342c69952086afb0f4094d0893d357e5c78752", - "sha256:bd640f60e8cecd74f0dc249713d433ace2ddc62b65ee07f96d358e0b152b6ea7" + "sha256:859b283c50bde45f5f97829f77a4674d1c1fcd88539364f1b28a37805cfd89c0", + "sha256:d8916d3f62a7b67ab353a952ce4ced6a1d2587dfe9ef8ebc30dd7c386751f289" ], "markers": "python_full_version >= '3.6.2'", - "version": "==3.0.29" + "version": "==3.0.30" }, "psycopg2-binary": { "hashes": [ @@ -665,6 +726,14 @@ ], "version": "==2022.1" }, + "redis": { + "hashes": [ + "sha256:a52d5694c9eb4292770084fa8c863f79367ca19884b329ab574d5cb2036b3e54", + "sha256:ddf27071df4adf3821c4f2ca59d67525c3a82e5f268bed97b813cb4fabf87880" + ], + "index": "pypi", + "version": "==4.3.4" + }, "requests": { "hashes": [ "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", @@ -729,11 +798,11 @@ }, "setuptools": { "hashes": [ - "sha256:68e45d17c9281ba25dc0104eadd2647172b3472d9e01f911efa57965e8d51a36", - "sha256:a43bdedf853c670e5fed28e5623403bad2f73cf02f9a2774e91def6bda8265a7" + "sha256:990a4f7861b31532871ab72331e755b5f14efbe52d336ea7f6118144dd478741", + "sha256:c1848f654aea2e3526d17fc3ce6aeaa5e7e24e66e645b5be2171f3f6b4e5a178" ], "markers": "python_version >= '3.7'", - "version": "==62.3.2" + "version": "==62.6.0" }, "six": { "hashes": [ @@ -753,26 +822,26 @@ }, "stack-data": { "hashes": [ - "sha256:45692d41bd633a9503a5195552df22b583caf16f0b27c4e58c98d88c8b648e12", - "sha256:999762f9c3132308789affa03e9271bbbe947bf78311851f4d485d8402ed858e" + "sha256:77bec1402dcd0987e9022326473fdbcc767304892a533ed8c29888dacb7dddbc", + "sha256:aa1d52d14d09c7a9a12bb740e6bdfffe0f5e8f4f9218d85e7c73a8c37f7ae38d" ], - "version": "==0.2.0" + "version": "==0.3.0" }, "toml": { "hashes": [ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.7'", "version": "==0.10.2" }, "traitlets": { "hashes": [ - "sha256:1530d04badddc6a73d50b7ee34667d4b96914da352109117b4280cb56523a51b", - "sha256:74803a1baa59af70f023671d86d5c7a834c931186df26d50d362ee6a1ff021fd" + "sha256:0bb9f1f9f017aa8ec187d8b1b2a7a6626a2a1d877116baba52a129bfa124f8e2", + "sha256:65fa18961659635933100db8ca120ef6220555286949774b9cfc106f941d1c7a" ], "markers": "python_version >= '3.7'", - "version": "==5.2.2.post1" + "version": "==5.3.0" }, "uritemplate": { "hashes": [ @@ -790,6 +859,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.9" }, + "vine": { + "hashes": [ + "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30", + "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e" + ], + "markers": "python_version >= '3.6'", + "version": "==5.0.0" + }, "wait-for-it": { "hashes": [ "sha256:9560f1c6fad73d801be1258bb7dd5f8288b0aebc0ff08e4c1eca026de5272190", @@ -877,13 +954,6 @@ } }, "develop": { - "appdirs": { - "hashes": [ - "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", - "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" - ], - "version": "==1.4.4" - }, "attrs": { "hashes": [ "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", @@ -894,11 +964,11 @@ }, "awscli": { "hashes": [ - "sha256:6828a5f3dd57a5dba73e988eb4e725eb77b90e1ebd17e40973197d07c05cb5c0", - "sha256:9a8cdba225275e0f4a4bff1f4aa9945e3af9796b63dd4b89e13763e02bd21e6a" + "sha256:603e259e16a790ce659c5d9aee25439d2e2f2bce3614c218a63b34e85d0afdec", + "sha256:8e814695a02cbbc0515160ec000336f3586d04e698e7c8066233fad3021ef051" ], "index": "pypi", - "version": "==1.24.1" + "version": "==1.25.20" }, "awscli-local": { "hashes": [ @@ -991,12 +1061,11 @@ }, "docutils": { "hashes": [ - "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", - "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", - "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", + "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.15.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.16" }, "factory-boy": { "hashes": [ @@ -1008,11 +1077,11 @@ }, "faker": { "hashes": [ - "sha256:0122b75e7960cbb1e2bbbf10ef9b8c183377878e38466854953539c6d822e7c0", - "sha256:fb95f956bac59c90f54543919d5c5ef41625e12a0773e5aa08c9b9c62ba58fb3" + "sha256:0297b7fc0f2458dfff8d5a92335c62fa25fb059f8cbaf7db580a0dd7177aff2e", + "sha256:b9f93ec97a70da79d43f497aa7b2b7d2bcd5d0c6d3ab7c102dde4193d0a38351" ], "markers": "python_version >= '3.6'", - "version": "==13.12.1" + "version": "==13.14.0" }, "flake8": { "hashes": [ @@ -1039,11 +1108,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700", - "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec" + "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670", + "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23" ], "markers": "python_version >= '3.7'", - "version": "==4.11.4" + "version": "==4.12.0" }, "inflection": { "hashes": [ @@ -1090,14 +1159,6 @@ ], "version": "==1.35" }, - "mako": { - "hashes": [ - "sha256:23aab11fdbbb0f1051b93793a58323ff937e98e34aece1c4219675122e57e4ba", - "sha256:9a7c7e922b87db3686210cf49d5d767033a41d4010b284e747682c92bddd8b39" - ], - "markers": "python_version >= '3.7'", - "version": "==1.2.0" - }, "markdown": { "hashes": [ "sha256:cbb516f16218e643d8e0a95b309f77eb118cb138d39a4f27851e6a63581db874", @@ -1282,19 +1343,19 @@ }, "pytest-factoryboy": { "hashes": [ - "sha256:f8bfffc4c95585fa18637f1a31878791c9e17da4556d19febfba769fda400e57", - "sha256:fc7dd2be0bd0c7557be2b28464265cdc3ec2feacc0f38ccef8f45ecb8cc054ad" + "sha256:37c6b3a17689b197025260ad06047d0dd8ec54a70556995b22617e9df1fd8797", + "sha256:8abcfe5396a2cadf0b2d6fc1ba607d05f191db915eac95733ab741cb30968a46" ], "index": "pypi", - "version": "==2.2.1" + "version": "==2.5.0" }, "pytest-mock": { "hashes": [ - "sha256:5112bd92cc9f186ee96e1a92efc84969ea494939c3aead39c50f421c4cc69534", - "sha256:6cff27cec936bf81dc5ee87f07132b807bcda51106b5ec4b90a04331cba76231" + "sha256:2c6d756d5d3bf98e2e80797a959ca7f81f479e7d1f5f571611b0fdd6d1745240", + "sha256:d989f11ca4a84479e288b0cd1e6769d6ad0d3d7743dcc75e460d1416a5f2135a" ], "index": "pypi", - "version": "==3.7.0" + "version": "==3.8.1" }, "python-dateutil": { "hashes": [ @@ -1352,7 +1413,7 @@ "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], - "markers": "python_version >= '3.5' and python_version < '4'", + "markers": "python_version >= '3.5' and python_version < '4.0'", "version": "==4.7.2" }, "s3transfer": { @@ -1403,34 +1464,34 @@ }, "watchdog": { "hashes": [ - "sha256:036ed15f7cd656351bf4e17244447be0a09a61aaa92014332d50719fc5973bc0", - "sha256:0c520009b8cce79099237d810aaa19bc920941c268578436b62013b2f0102320", - "sha256:0fb60c7d31474b21acba54079ce9ff0136411183e9a591369417cddb1d7d00d7", - "sha256:156ec3a94695ea68cfb83454b98754af6e276031ba1ae7ae724dc6bf8973b92a", - "sha256:1ae17b6be788fb8e4d8753d8d599de948f0275a232416e16436363c682c6f850", - "sha256:1e5d0fdfaa265c29dc12621913a76ae99656cf7587d03950dfeb3595e5a26102", - "sha256:24dedcc3ce75e150f2a1d704661f6879764461a481ba15a57dc80543de46021c", - "sha256:2962628a8777650703e8f6f2593065884c602df7bae95759b2df267bd89b2ef5", - "sha256:47598fe6713fc1fee86b1ca85c9cbe77e9b72d002d6adeab9c3b608f8a5ead10", - "sha256:4978db33fc0934c92013ee163a9db158ec216099b69fce5aec790aba704da412", - "sha256:5e2e51c53666850c3ecffe9d265fc5d7351db644de17b15e9c685dd3cdcd6f97", - "sha256:676263bee67b165f16b05abc52acc7a94feac5b5ab2449b491f1a97638a79277", - "sha256:68dbe75e0fa1ba4d73ab3f8e67b21770fbed0651d32ce515cd38919a26873266", - "sha256:6d03149126864abd32715d4e9267d2754cede25a69052901399356ad3bc5ecff", - "sha256:6ddf67bc9f413791072e3afb466e46cc72c6799ba73dea18439b412e8f2e3257", - "sha256:746e4c197ec1083581bb1f64d07d1136accf03437badb5ff8fcb862565c193b2", - "sha256:7721ac736170b191c50806f43357407138c6748e4eb3e69b071397f7f7aaeedd", - "sha256:88ef3e8640ef0a64b7ad7394b0f23384f58ac19dd759da7eaa9bc04b2898943f", - "sha256:aa68d2d9a89d686fae99d28a6edf3b18595e78f5adf4f5c18fbfda549ac0f20c", - "sha256:b962de4d7d92ff78fb2dbc6a0cb292a679dea879a0eb5568911484d56545b153", - "sha256:ce7376aed3da5fd777483fe5ebc8475a440c6d18f23998024f832134b2938e7b", - "sha256:ddde157dc1447d8130cb5b8df102fad845916fe4335e3d3c3f44c16565becbb7", - "sha256:efcc8cbc1b43902571b3dce7ef53003f5b97fe4f275fe0489565fc6e2ebe3314", - "sha256:f9ee4c6bf3a1b2ed6be90a2d78f3f4bbd8105b6390c04a86eb48ed67bbfa0b0b", - "sha256:fed4de6e45a4f16e4046ea00917b4fe1700b97244e5d114f594b4a1b9de6bed8" + "sha256:083171652584e1b8829581f965b9b7723ca5f9a2cd7e20271edf264cfd7c1412", + "sha256:117ffc6ec261639a0209a3252546b12800670d4bf5f84fbd355957a0595fe654", + "sha256:186f6c55abc5e03872ae14c2f294a153ec7292f807af99f57611acc8caa75306", + "sha256:195fc70c6e41237362ba720e9aaf394f8178bfc7fa68207f112d108edef1af33", + "sha256:226b3c6c468ce72051a4c15a4cc2ef317c32590d82ba0b330403cafd98a62cfd", + "sha256:247dcf1df956daa24828bfea5a138d0e7a7c98b1a47cf1fa5b0c3c16241fcbb7", + "sha256:255bb5758f7e89b1a13c05a5bceccec2219f8995a3a4c4d6968fe1de6a3b2892", + "sha256:43ce20ebb36a51f21fa376f76d1d4692452b2527ccd601950d69ed36b9e21609", + "sha256:4f4e1c4aa54fb86316a62a87b3378c025e228178d55481d30d857c6c438897d6", + "sha256:5952135968519e2447a01875a6f5fc8c03190b24d14ee52b0f4b1682259520b1", + "sha256:64a27aed691408a6abd83394b38503e8176f69031ca25d64131d8d640a307591", + "sha256:6b17d302850c8d412784d9246cfe8d7e3af6bcd45f958abb2d08a6f8bedf695d", + "sha256:70af927aa1613ded6a68089a9262a009fbdf819f46d09c1a908d4b36e1ba2b2d", + "sha256:7a833211f49143c3d336729b0020ffd1274078e94b0ae42e22f596999f50279c", + "sha256:8250546a98388cbc00c3ee3cc5cf96799b5a595270dfcfa855491a64b86ef8c3", + "sha256:97f9752208f5154e9e7b76acc8c4f5a58801b338de2af14e7e181ee3b28a5d39", + "sha256:9f05a5f7c12452f6a27203f76779ae3f46fa30f1dd833037ea8cbc2887c60213", + "sha256:a735a990a1095f75ca4f36ea2ef2752c99e6ee997c46b0de507ba40a09bf7330", + "sha256:ad576a565260d8f99d97f2e64b0f97a48228317095908568a9d5c786c829d428", + "sha256:b530ae007a5f5d50b7fbba96634c7ee21abec70dc3e7f0233339c81943848dc1", + "sha256:bfc4d351e6348d6ec51df007432e6fe80adb53fd41183716017026af03427846", + "sha256:d3dda00aca282b26194bdd0adec21e4c21e916956d972369359ba63ade616153", + "sha256:d9820fe47c20c13e3c9dd544d3706a2a26c02b2b43c993b62fcd8011bcc0adb3", + "sha256:ed80a1628cee19f5cfc6bb74e173f1b4189eb532e705e2a13e3250312a62e0c9", + "sha256:ee3e38a6cc050a8830089f79cbec8a3878ec2fe5160cdb2dc8ccb6def8552658" ], "markers": "python_version >= '3.6'", - "version": "==2.1.8" + "version": "==2.1.9" }, "zipp": { "hashes": [ diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index e41401e98a..68cfe47114 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -80,6 +80,13 @@ services: - clamav-rest - localstack - postgres + - redis-server + + redis-server: + image: "redis:alpine" + command: redis-server + ports: + - "6380:6379" volumes: localstack_data: diff --git a/tdrs-backend/tdpservice/scheduling/__init__.py b/tdrs-backend/tdpservice/scheduling/__init__.py new file mode 100644 index 0000000000..71ff1a101f --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from tdpservice.scheduling.celery import app as celery_app diff --git a/tdrs-backend/tdpservice/scheduling/celery.py b/tdrs-backend/tdpservice/scheduling/celery.py new file mode 100644 index 0000000000..000e9a4839 --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/celery.py @@ -0,0 +1,23 @@ +from __future__ import absolute_import +import os +from celery import Celery +from django.conf import settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scheduling.settings') +app = Celery('scheduling') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) + + +@app.task(name="add_two_numbers") +def add(x,y): + return x+y diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 4dc1b6e10a..25744eb341 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -397,3 +397,11 @@ class Common(Configuration): 'AMS_CLIENT_SECRET', '' ) + + # CELERY CONFIG + BROKER_URL = 'redis://redis-server:6380' + CELERY_RESULT_BACKEND = 'redis://redis-server:6380' + CELERY_ACCEPT_CONTENT = ['application/json'] + CELERY_TASK_SERIALIZER = 'json' + CELERY_RESULT_SERIALIZER = 'json' + CELERY_TIMEZONE = 'Africa/Nairobi' From 6917bbb48f50bdacb77868f45d96ba466ce04275 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 1 Jul 2022 13:20:19 -0700 Subject: [PATCH 003/113] 1133: redis works --- tdrs-backend/__init__.py | 0 tdrs-backend/docker-compose.yml | 2 +- tdrs-backend/tdpservice/__init__.py | 10 ++++++ tdrs-backend/tdpservice/celery.py | 31 +++++++++++++++++++ .../tdpservice/scheduling/__init__.py | 5 --- tdrs-backend/tdpservice/scheduling/celery.py | 23 -------------- tdrs-backend/tdpservice/settings/__init__.py | 1 + tdrs-backend/tdpservice/settings/common.py | 6 ++-- tdrs-backend/tdpservice/settings/local.py | 1 + 9 files changed, 47 insertions(+), 32 deletions(-) create mode 100644 tdrs-backend/__init__.py create mode 100644 tdrs-backend/tdpservice/celery.py delete mode 100644 tdrs-backend/tdpservice/scheduling/__init__.py delete mode 100644 tdrs-backend/tdpservice/scheduling/celery.py diff --git a/tdrs-backend/__init__.py b/tdrs-backend/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 68cfe47114..8547b656db 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -86,7 +86,7 @@ services: image: "redis:alpine" command: redis-server ports: - - "6380:6379" + - "6379:6379" volumes: localstack_data: diff --git a/tdrs-backend/tdpservice/__init__.py b/tdrs-backend/tdpservice/__init__.py index e69de29bb2..fd90d39eff 100644 --- a/tdrs-backend/tdpservice/__init__.py +++ b/tdrs-backend/tdpservice/__init__.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import +import logging +import os +from django.conf import settings + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from tdpservice.celery import app as celery_app + +__all__ = ['celery_app'] diff --git a/tdrs-backend/tdpservice/celery.py b/tdrs-backend/tdpservice/celery.py new file mode 100644 index 0000000000..a0d9c87c81 --- /dev/null +++ b/tdrs-backend/tdpservice/celery.py @@ -0,0 +1,31 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tdpservice.settings.common') + +app = Celery('tdpservice') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + print(f'Request: {self.request!r}') + + +from celery import shared_task + + +@shared_task +def add(x, y): + return x + y + diff --git a/tdrs-backend/tdpservice/scheduling/__init__.py b/tdrs-backend/tdpservice/scheduling/__init__.py deleted file mode 100644 index 71ff1a101f..0000000000 --- a/tdrs-backend/tdpservice/scheduling/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import absolute_import - -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from tdpservice.scheduling.celery import app as celery_app diff --git a/tdrs-backend/tdpservice/scheduling/celery.py b/tdrs-backend/tdpservice/scheduling/celery.py deleted file mode 100644 index 000e9a4839..0000000000 --- a/tdrs-backend/tdpservice/scheduling/celery.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import absolute_import -import os -from celery import Celery -from django.conf import settings - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'scheduling.settings') -app = Celery('scheduling') - -# Using a string here means the worker will not have to -# pickle the object when using Windows. -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) - - -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) - - -@app.task(name="add_two_numbers") -def add(x,y): - return x+y diff --git a/tdrs-backend/tdpservice/settings/__init__.py b/tdrs-backend/tdpservice/settings/__init__.py index e69de29bb2..8b13789179 100755 --- a/tdrs-backend/tdpservice/settings/__init__.py +++ b/tdrs-backend/tdpservice/settings/__init__.py @@ -0,0 +1 @@ + diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 25744eb341..41a6c27f7c 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -69,7 +69,7 @@ class Common(Configuration): "corsheaders.middleware.CorsMiddleware", "tdpservice.users.api.middleware.AuthUpdateMiddleware", "csp.middleware.CSPMiddleware", - "tdpservice.middleware.NoCacheMiddleware", + "tdpservice.middleware.NoCacheMiddleware" ) APP_NAME = "dev" @@ -399,8 +399,8 @@ class Common(Configuration): ) # CELERY CONFIG - BROKER_URL = 'redis://redis-server:6380' - CELERY_RESULT_BACKEND = 'redis://redis-server:6380' + CELERY_BROKER_URL = "redis://redis-server:6379/0" + CELERY_RESULT_BACKEND = "redis://redis-server:6379/0" CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 83be95bd3b..967e192a92 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -43,3 +43,4 @@ class Local(Common): 'level': 'DEBUG', 'handlers': ['console'] } + From d880c6f0f686f1c523b7b9f047a87b4cad252a4d Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 1 Jul 2022 15:05:20 -0700 Subject: [PATCH 004/113] 1133: redis works with celery installed in scheduling app --- tdrs-backend/tdpservice/__init__.py | 10 ---------- tdrs-backend/tdpservice/scheduling/__init__.py | 10 ++++++++++ tdrs-backend/tdpservice/{ => scheduling}/celery.py | 0 3 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 tdrs-backend/tdpservice/scheduling/__init__.py rename tdrs-backend/tdpservice/{ => scheduling}/celery.py (100%) diff --git a/tdrs-backend/tdpservice/__init__.py b/tdrs-backend/tdpservice/__init__.py index fd90d39eff..e69de29bb2 100644 --- a/tdrs-backend/tdpservice/__init__.py +++ b/tdrs-backend/tdpservice/__init__.py @@ -1,10 +0,0 @@ -from __future__ import absolute_import -import logging -import os -from django.conf import settings - -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from tdpservice.celery import app as celery_app - -__all__ = ['celery_app'] diff --git a/tdrs-backend/tdpservice/scheduling/__init__.py b/tdrs-backend/tdpservice/scheduling/__init__.py new file mode 100644 index 0000000000..afda9ae5fc --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/__init__.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import +import logging +import os +from django.conf import settings + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from tdpservice.scheduling.celery import app as celery_app + +__all__ = ['celery_app'] diff --git a/tdrs-backend/tdpservice/celery.py b/tdrs-backend/tdpservice/scheduling/celery.py similarity index 100% rename from tdrs-backend/tdpservice/celery.py rename to tdrs-backend/tdpservice/scheduling/celery.py From c0068618f8d1123915f81756e4f5c1240b05d298 Mon Sep 17 00:00:00 2001 From: msohanifr Date: Tue, 5 Jul 2022 11:11:02 -0500 Subject: [PATCH 005/113] corrections --- tdrs-backend/Dockerfile | 2 -- tdrs-backend/__init__.py | 0 tdrs-backend/tdpservice/settings/__init__.py | 1 - tdrs-backend/tdpservice/settings/common.py | 2 +- 4 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 tdrs-backend/__init__.py diff --git a/tdrs-backend/Dockerfile b/tdrs-backend/Dockerfile index 1704c2c47f..f43ec2a41b 100644 --- a/tdrs-backend/Dockerfile +++ b/tdrs-backend/Dockerfile @@ -16,8 +16,6 @@ RUN apt-get -y update RUN apt-get -y upgrade # Install a new package: RUN apt-get install -y gcc && apt-get install -y graphviz && apt-get install -y graphviz-dev -# Install Redis -# RUN apt-get install -y redis-server # Install pipenv RUN pip install --upgrade pip pipenv && pipenv install --dev --system --deploy diff --git a/tdrs-backend/__init__.py b/tdrs-backend/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tdrs-backend/tdpservice/settings/__init__.py b/tdrs-backend/tdpservice/settings/__init__.py index 8b13789179..e69de29bb2 100755 --- a/tdrs-backend/tdpservice/settings/__init__.py +++ b/tdrs-backend/tdpservice/settings/__init__.py @@ -1 +0,0 @@ - diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 41a6c27f7c..43745c0cbe 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -404,4 +404,4 @@ class Common(Configuration): CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' - CELERY_TIMEZONE = 'Africa/Nairobi' + CELERY_TIMEZONE = 'UTC' From c595ffb1e732b08fff79156c2fd1f7b90b1011b5 Mon Sep 17 00:00:00 2001 From: msohanifr Date: Wed, 6 Jul 2022 01:54:15 -0500 Subject: [PATCH 006/113] changes to use django-configurations --- .../tdpservice/scheduling/__init__.py | 1 - tdrs-backend/tdpservice/scheduling/celery.py | 28 +++++++++---------- tdrs-backend/tdpservice/settings/__init__.py | 1 + tdrs-backend/tdpservice/settings/common.py | 9 ------ tdrs-backend/tdpservice/settings/local.py | 8 ++++++ 5 files changed, 23 insertions(+), 24 deletions(-) mode change 100755 => 100644 tdrs-backend/tdpservice/settings/__init__.py diff --git a/tdrs-backend/tdpservice/scheduling/__init__.py b/tdrs-backend/tdpservice/scheduling/__init__.py index afda9ae5fc..109573bb52 100644 --- a/tdrs-backend/tdpservice/scheduling/__init__.py +++ b/tdrs-backend/tdpservice/scheduling/__init__.py @@ -6,5 +6,4 @@ # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from tdpservice.scheduling.celery import app as celery_app - __all__ = ['celery_app'] diff --git a/tdrs-backend/tdpservice/scheduling/celery.py b/tdrs-backend/tdpservice/scheduling/celery.py index a0d9c87c81..9cecd5e47f 100644 --- a/tdrs-backend/tdpservice/scheduling/celery.py +++ b/tdrs-backend/tdpservice/scheduling/celery.py @@ -1,31 +1,31 @@ +from __future__ import absolute_import import os - from celery import Celery +from django.conf import settings # Set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tdpservice.settings.common') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.local") +os.environ.setdefault("DJANGO_CONFIGURATION", "Local") -app = Celery('tdpservice') +import configurations +configurations.setup() +app = Celery('settings') # Using a string here means the worker doesn't have to serialize # the configuration object to child processes. # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') -# Load task modules from all registered Django apps. -app.autodiscover_tasks() - - -@app.task(bind=True) -def debug_task(self): - print(f'Request: {self.request!r}') - - -from celery import shared_task +# Load task modules from all registered Django apps. +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) -@shared_task +@app.task def add(x, y): return x + y +@app.task() +def debug_task(self): + print('Request: {0!r}'.format(self.request)) + diff --git a/tdrs-backend/tdpservice/settings/__init__.py b/tdrs-backend/tdpservice/settings/__init__.py old mode 100755 new mode 100644 index e69de29bb2..8b13789179 --- a/tdrs-backend/tdpservice/settings/__init__.py +++ b/tdrs-backend/tdpservice/settings/__init__.py @@ -0,0 +1 @@ + diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 43745c0cbe..61209e60c2 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -12,7 +12,6 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - def get_required_env_var_setting( env_var_name: str, setting_name: Optional[str] = None @@ -397,11 +396,3 @@ class Common(Configuration): 'AMS_CLIENT_SECRET', '' ) - - # CELERY CONFIG - CELERY_BROKER_URL = "redis://redis-server:6379/0" - CELERY_RESULT_BACKEND = "redis://redis-server:6379/0" - CELERY_ACCEPT_CONTENT = ['application/json'] - CELERY_TASK_SERIALIZER = 'json' - CELERY_RESULT_SERIALIZER = 'json' - CELERY_TIMEZONE = 'UTC' diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 967e192a92..2b3008e022 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -44,3 +44,11 @@ class Local(Common): 'handlers': ['console'] } + # CELERY CONFIG + CELERY_BROKER_URL = "redis://redis-server:6379" + CELERY_RESULT_BACKEND = "redis://redis-server:6379" + CELERY_ACCEPT_CONTENT = ['application/json'] + CELERY_TASK_SERIALIZER = 'json' + CELERY_RESULT_SERIALIZER = 'json' + CELERY_TIMEZONE = 'UTC' + From 22a20e27bec5c641803f77675057b2247665b5cf Mon Sep 17 00:00:00 2001 From: msohanifr Date: Wed, 6 Jul 2022 02:08:40 -0500 Subject: [PATCH 007/113] remove white space --- tdrs-backend/tdpservice/settings/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/__init__.py b/tdrs-backend/tdpservice/settings/__init__.py index 8b13789179..e69de29bb2 100644 --- a/tdrs-backend/tdpservice/settings/__init__.py +++ b/tdrs-backend/tdpservice/settings/__init__.py @@ -1 +0,0 @@ - From 6cf5549de2382296098445b9d30f521a77b58985 Mon Sep 17 00:00:00 2001 From: msohanifr Date: Wed, 6 Jul 2022 02:10:16 -0500 Subject: [PATCH 008/113] delete __init__ file --- tdrs-backend/tdpservice/settings/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tdrs-backend/tdpservice/settings/__init__.py diff --git a/tdrs-backend/tdpservice/settings/__init__.py b/tdrs-backend/tdpservice/settings/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From 51c818cdbb9dd1369064e259ca2bd826781f7a23 Mon Sep 17 00:00:00 2001 From: msohanifr Date: Wed, 6 Jul 2022 02:12:53 -0500 Subject: [PATCH 009/113] added __init__ --- tdrs-backend/tdpservice/settings/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tdrs-backend/tdpservice/settings/__init__.py diff --git a/tdrs-backend/tdpservice/settings/__init__.py b/tdrs-backend/tdpservice/settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From c9c7b7bbea5292b548a12826999c1ce53cb7edfe Mon Sep 17 00:00:00 2001 From: msohanifr Date: Mon, 11 Jul 2022 09:40:00 -0400 Subject: [PATCH 010/113] added a new app, and sub task --- tdrs-backend/__init__.py | 0 tdrs-backend/tasks/__init__.py | 0 tdrs-backend/tasks/migrations/__init__.py | 0 tdrs-backend/tasks/models.py | 3 +++ tdrs-backend/tasks/tests.py | 3 +++ tdrs-backend/tasks/views.py | 3 +++ tdrs-backend/tdpservice/scheduling/__init__.py | 9 --------- tdrs-backend/tdpservice/scheduling/admin.py | 3 +++ tdrs-backend/tdpservice/scheduling/apps.py | 6 ++++++ tdrs-backend/tdpservice/scheduling/tasks.py | 6 ++++++ tdrs-backend/tdpservice/settings/__init__.py | 9 +++++++++ .../tdpservice/{scheduling => settings}/celery.py | 12 +++--------- tdrs-backend/tdpservice/settings/common.py | 1 + 13 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 tdrs-backend/__init__.py create mode 100644 tdrs-backend/tasks/__init__.py create mode 100644 tdrs-backend/tasks/migrations/__init__.py create mode 100644 tdrs-backend/tasks/models.py create mode 100644 tdrs-backend/tasks/tests.py create mode 100644 tdrs-backend/tasks/views.py create mode 100644 tdrs-backend/tdpservice/scheduling/admin.py create mode 100644 tdrs-backend/tdpservice/scheduling/apps.py create mode 100644 tdrs-backend/tdpservice/scheduling/tasks.py rename tdrs-backend/tdpservice/{scheduling => settings}/celery.py (84%) diff --git a/tdrs-backend/__init__.py b/tdrs-backend/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tdrs-backend/tasks/__init__.py b/tdrs-backend/tasks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tdrs-backend/tasks/migrations/__init__.py b/tdrs-backend/tasks/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tdrs-backend/tasks/models.py b/tdrs-backend/tasks/models.py new file mode 100644 index 0000000000..71a8362390 --- /dev/null +++ b/tdrs-backend/tasks/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/tdrs-backend/tasks/tests.py b/tdrs-backend/tasks/tests.py new file mode 100644 index 0000000000..7ce503c2dd --- /dev/null +++ b/tdrs-backend/tasks/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/tdrs-backend/tasks/views.py b/tdrs-backend/tasks/views.py new file mode 100644 index 0000000000..91ea44a218 --- /dev/null +++ b/tdrs-backend/tasks/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/tdrs-backend/tdpservice/scheduling/__init__.py b/tdrs-backend/tdpservice/scheduling/__init__.py index 109573bb52..e69de29bb2 100644 --- a/tdrs-backend/tdpservice/scheduling/__init__.py +++ b/tdrs-backend/tdpservice/scheduling/__init__.py @@ -1,9 +0,0 @@ -from __future__ import absolute_import -import logging -import os -from django.conf import settings - -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from tdpservice.scheduling.celery import app as celery_app -__all__ = ['celery_app'] diff --git a/tdrs-backend/tdpservice/scheduling/admin.py b/tdrs-backend/tdpservice/scheduling/admin.py new file mode 100644 index 0000000000..8c38f3f3da --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/tdrs-backend/tdpservice/scheduling/apps.py b/tdrs-backend/tdpservice/scheduling/apps.py new file mode 100644 index 0000000000..8e7fd7a29a --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class TasksConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'tdpservice.scheduling' diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py new file mode 100644 index 0000000000..189e36ddcc --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -0,0 +1,6 @@ +from __future__ import absolute_import +from celery import shared_task + +@shared_task +def upload(a,b): + return a*b diff --git a/tdrs-backend/tdpservice/settings/__init__.py b/tdrs-backend/tdpservice/settings/__init__.py index e69de29bb2..9603df3969 100644 --- a/tdrs-backend/tdpservice/settings/__init__.py +++ b/tdrs-backend/tdpservice/settings/__init__.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import +import logging +import os +from django.conf import settings + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app +__all__ = ['celery_app'] diff --git a/tdrs-backend/tdpservice/scheduling/celery.py b/tdrs-backend/tdpservice/settings/celery.py similarity index 84% rename from tdrs-backend/tdpservice/scheduling/celery.py rename to tdrs-backend/tdpservice/settings/celery.py index 9cecd5e47f..d3245705f5 100644 --- a/tdrs-backend/tdpservice/scheduling/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import os -from celery import Celery +from celery import Celery, shared_task from django.conf import settings # Set the default Django settings module for the 'celery' program. @@ -17,15 +17,9 @@ # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') - # Load task modules from all registered Django apps. -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) - -@app.task -def add(x, y): - return x + y +app.autodiscover_tasks() -@app.task() +@app.task(bind=True) def debug_task(self): print('Request: {0!r}'.format(self.request)) - diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 61209e60c2..bc8c27d3e9 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -54,6 +54,7 @@ class Common(Configuration): "tdpservice.stts", "tdpservice.data_files", "tdpservice.security", + "tdpservice.scheduling", ) # https://docs.djangoproject.com/en/2.0/topics/http/middleware/ From 81c8ea10dd66c2e5040023f06d996fcfaeb0d94d Mon Sep 17 00:00:00 2001 From: msohanifr Date: Mon, 11 Jul 2022 12:35:08 -0400 Subject: [PATCH 011/113] added django_celery_beat --- tdrs-backend/Pipfile | 2 + tdrs-backend/Pipfile.lock | 351 ++++++++++++++------- tdrs-backend/docker-compose.yml | 1 + tdrs-backend/tdpservice/settings/common.py | 1 + 4 files changed, 236 insertions(+), 119 deletions(-) diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index cc3990c416..c52216e805 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -47,6 +47,8 @@ wait-for-it = "==2.2.0" requests-mock = "==1.9.3" celery = "*" redis = "*" +flower = "1.1.0" +django-celery-beat = "2.3.0" [requires] python_version = "3.10.4" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index 8cd5345cc2..16b5b997fa 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "803728542591a1713c7e67f0ae6c3ef8b2057a32c5ff1c4a43d34c45654c5796" + "sha256": "50b95aa34c5149af5d8d39f33b0b20fa505925e63e641fae04d04d81e9d1b38c" }, "pipfile-spec": 6, "requires": { @@ -24,6 +24,14 @@ "markers": "python_version >= '3.6'", "version": "==5.1.1" }, + "appnope": { + "hashes": [ + "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", + "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" + ], + "markers": "sys_platform == 'darwin'", + "version": "==0.1.3" + }, "asgiref": { "hashes": [ "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4", @@ -95,58 +103,72 @@ }, "cffi": { "hashes": [ - "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", - "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", - "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", - "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", - "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", - "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", - "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", - "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", - "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", - "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", - "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", - "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", - "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", - "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", - "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", - "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", - "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", - "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", - "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", - "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", - "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", - "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", - "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", - "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", - "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", - "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", - "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", - "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", - "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", - "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", - "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", - "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", - "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", - "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", - "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", - "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", - "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", - "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", - "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", - "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", - "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", - "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", - "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", - "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", - "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", - "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", - "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", - "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", - "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", - "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" - ], - "version": "==1.15.0" + "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", + "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", + "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", + "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", + "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", + "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", + "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", + "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", + "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", + "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", + "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", + "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", + "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", + "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", + "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", + "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", + "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", + "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", + "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", + "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", + "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", + "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", + "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", + "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", + "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", + "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", + "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", + "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", + "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", + "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", + "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", + "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", + "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", + "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", + "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", + "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", + "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", + "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", + "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", + "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", + "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", + "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", + "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", + "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", + "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", + "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", + "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", + "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", + "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", + "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", + "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", + "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", + "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", + "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", + "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", + "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", + "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", + "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", + "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", + "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", + "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", + "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", + "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", + "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" + ], + "version": "==1.15.1" }, "charset-normalizer": { "hashes": [ @@ -225,7 +247,7 @@ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186" ], - "markers": "python_version >= '3.7'", + "markers": "python_version >= '3.5'", "version": "==5.1.1" }, "deprecated": { @@ -268,6 +290,14 @@ "index": "pypi", "version": "==1.0.2" }, + "django-celery-beat": { + "hashes": [ + "sha256:ba24f86ed956ba0ee2b43237a49303ebf5aa7e0fb0ccb7e0542b7e31a463dcba", + "sha256:ff694e1d30f84de3d2b78822c90005c6c7c1462768e845ae162af71a6a46f1eb" + ], + "index": "pypi", + "version": "==2.3.0" + }, "django-colorfield": { "hashes": [ "sha256:08d093d5b31b599b681665ab97b957045c4a4c0b93cfc434bd257378a7193c96", @@ -343,6 +373,14 @@ "index": "pypi", "version": "==1.12.3" }, + "django-timezone-field": { + "hashes": [ + "sha256:15746ed367a5a32eda76cfa2886eeec1de8cda79f519b7c5e12f87ed7cdbd663", + "sha256:199f211082eeac7e83563929b8ce41399c1c0f00dfc2f36bc00bea381027eaaa" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==5.0" + }, "django-unique-upload": { "hashes": [ "sha256:78af8e8948a640dc3ef11f16189ae8dfd81293836d991913fbf40bf5111f43a9", @@ -374,6 +412,14 @@ ], "version": "==0.8.3" }, + "flower": { + "hashes": [ + "sha256:77be4bece7330893703e8cca3f3baddba356a0019186594a961e3968289da7ec", + "sha256:f920ca2902d7539fc180ab15e51f1d918355e5cc23da854ff9d59c6e5a1725b6" + ], + "index": "pypi", + "version": "==1.1.0" + }, "gunicorn": { "hashes": [ "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e", @@ -382,6 +428,14 @@ "index": "pypi", "version": "==20.1.0" }, + "humanize": { + "hashes": [ + "sha256:2bc1fdd831cd00557d3010abdd84d3e41b4a96703a3eaf6c24ee290b26b75a44", + "sha256:bed628920d45cd5018abb095710f0c03a8336d6ac0790e7647c6a328f3880b81" + ], + "markers": "python_version >= '3.7'", + "version": "==4.2.3" + }, "idna": { "hashes": [ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", @@ -410,7 +464,7 @@ "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1", "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd" ], - "markers": "python_version >= '3.7'", + "markers": "python_version >= '3.8'", "version": "==8.4.0" }, "itypes": { @@ -554,47 +608,75 @@ }, "pillow": { "hashes": [ - "sha256:088df396b047477dd1bbc7de6e22f58400dae2f21310d9e2ec2933b2ef7dfa4f", - "sha256:09e67ef6e430f90caa093528bd758b0616f8165e57ed8d8ce014ae32df6a831d", - "sha256:0b4d5ad2cd3a1f0d1df882d926b37dbb2ab6c823ae21d041b46910c8f8cd844b", - "sha256:0b525a356680022b0af53385944026d3486fc8c013638cf9900eb87c866afb4c", - "sha256:1d4331aeb12f6b3791911a6da82de72257a99ad99726ed6b63f481c0184b6fb9", - "sha256:20d514c989fa28e73a5adbddd7a171afa5824710d0ab06d4e1234195d2a2e546", - "sha256:2b291cab8a888658d72b575a03e340509b6b050b62db1f5539dd5cd18fd50578", - "sha256:3f6c1716c473ebd1649663bf3b42702d0d53e27af8b64642be0dd3598c761fb1", - "sha256:42dfefbef90eb67c10c45a73a9bc1599d4dac920f7dfcbf4ec6b80cb620757fe", - "sha256:488f3383cf5159907d48d32957ac6f9ea85ccdcc296c14eca1a4e396ecc32098", - "sha256:4d45dbe4b21a9679c3e8b3f7f4f42a45a7d3ddff8a4a16109dff0e1da30a35b2", - "sha256:53c27bd452e0f1bc4bfed07ceb235663a1df7c74df08e37fd6b03eb89454946a", - "sha256:55e74faf8359ddda43fee01bffbc5bd99d96ea508d8a08c527099e84eb708f45", - "sha256:59789a7d06c742e9d13b883d5e3569188c16acb02eeed2510fd3bfdbc1bd1530", - "sha256:5b650dbbc0969a4e226d98a0b440c2f07a850896aed9266b6fedc0f7e7834108", - "sha256:66daa16952d5bf0c9d5389c5e9df562922a59bd16d77e2a276e575d32e38afd1", - "sha256:6e760cf01259a1c0a50f3c845f9cad1af30577fd8b670339b1659c6d0e7a41dd", - "sha256:7502539939b53d7565f3d11d87c78e7ec900d3c72945d4ee0e2f250d598309a0", - "sha256:769a7f131a2f43752455cc72f9f7a093c3ff3856bf976c5fb53a59d0ccc704f6", - "sha256:7c150dbbb4a94ea4825d1e5f2c5501af7141ea95825fadd7829f9b11c97aaf6c", - "sha256:8844217cdf66eabe39567118f229e275f0727e9195635a15e0e4b9227458daaf", - "sha256:8a66fe50386162df2da701b3722781cbe90ce043e7d53c1fd6bd801bca6b48d4", - "sha256:9370d6744d379f2de5d7fa95cdbd3a4d92f0b0ef29609b4b1687f16bc197063d", - "sha256:937a54e5694684f74dcbf6e24cc453bfc5b33940216ddd8f4cd8f0f79167f765", - "sha256:9c857532c719fb30fafabd2371ce9b7031812ff3889d75273827633bca0c4602", - "sha256:a4165205a13b16a29e1ac57efeee6be2dfd5b5408122d59ef2145bc3239fa340", - "sha256:b3fe2ff1e1715d4475d7e2c3e8dabd7c025f4410f79513b4ff2de3d51ce0fa9c", - "sha256:b6617221ff08fbd3b7a811950b5c3f9367f6e941b86259843eab77c8e3d2b56b", - "sha256:b761727ed7d593e49671d1827044b942dd2f4caae6e51bab144d4accf8244a84", - "sha256:baf3be0b9446a4083cc0c5bb9f9c964034be5374b5bc09757be89f5d2fa247b8", - "sha256:c17770a62a71718a74b7548098a74cd6880be16bcfff5f937f900ead90ca8e92", - "sha256:c67db410508b9de9c4694c57ed754b65a460e4812126e87f5052ecf23a011a54", - "sha256:d78ca526a559fb84faaaf84da2dd4addef5edb109db8b81677c0bb1aad342601", - "sha256:e9ed59d1b6ee837f4515b9584f3d26cf0388b742a11ecdae0d9237a94505d03a", - "sha256:f054b020c4d7e9786ae0404278ea318768eb123403b18453e28e47cdb7a0a4bf", - "sha256:f372d0f08eff1475ef426344efe42493f71f377ec52237bf153c5713de987251", - "sha256:f3f6a6034140e9e17e9abc175fc7a266a6e63652028e157750bd98e804a8ed9a", - "sha256:ffde4c6fabb52891d81606411cbfaf77756e3b561b566efd270b3ed3791fde4e" + "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927", + "sha256:030e3460861488e249731c3e7ab59b07c7853838ff3b8e16aac9561bb345da14", + "sha256:0ed2c4ef2451de908c90436d6e8092e13a43992f1860275b4d8082667fbb2ffc", + "sha256:136659638f61a251e8ed3b331fc6ccd124590eeff539de57c5f80ef3a9594e58", + "sha256:13b725463f32df1bfeacbf3dd197fb358ae8ebcd8c5548faa75126ea425ccb60", + "sha256:1536ad017a9f789430fb6b8be8bf99d2f214c76502becc196c6f2d9a75b01b76", + "sha256:15928f824870535c85dbf949c09d6ae7d3d6ac2d6efec80f3227f73eefba741c", + "sha256:17d4cafe22f050b46d983b71c707162d63d796a1235cdf8b9d7a112e97b15bac", + "sha256:1802f34298f5ba11d55e5bb09c31997dc0c6aed919658dfdf0198a2fe75d5490", + "sha256:1cc1d2451e8a3b4bfdb9caf745b58e6c7a77d2e469159b0d527a4554d73694d1", + "sha256:1fd6f5e3c0e4697fa7eb45b6e93996299f3feee73a3175fa451f49a74d092b9f", + "sha256:254164c57bab4b459f14c64e93df11eff5ded575192c294a0c49270f22c5d93d", + "sha256:2ad0d4df0f5ef2247e27fc790d5c9b5a0af8ade9ba340db4a73bb1a4a3e5fb4f", + "sha256:2c58b24e3a63efd22554c676d81b0e57f80e0a7d3a5874a7e14ce90ec40d3069", + "sha256:2d33a11f601213dcd5718109c09a52c2a1c893e7461f0be2d6febc2879ec2402", + "sha256:337a74fd2f291c607d220c793a8135273c4c2ab001b03e601c36766005f36885", + "sha256:37ff6b522a26d0538b753f0b4e8e164fdada12db6c6f00f62145d732d8a3152e", + "sha256:3d1f14f5f691f55e1b47f824ca4fdcb4b19b4323fe43cc7bb105988cad7496be", + "sha256:408673ed75594933714482501fe97e055a42996087eeca7e5d06e33218d05aa8", + "sha256:4134d3f1ba5f15027ff5c04296f13328fecd46921424084516bdb1b2548e66ff", + "sha256:4ad2f835e0ad81d1689f1b7e3fbac7b01bb8777d5a985c8962bedee0cc6d43da", + "sha256:50dff9cc21826d2977ef2d2a205504034e3a4563ca6f5db739b0d1026658e004", + "sha256:510cef4a3f401c246cfd8227b300828715dd055463cdca6176c2e4036df8bd4f", + "sha256:5aed7dde98403cd91d86a1115c78d8145c83078e864c1de1064f52e6feb61b20", + "sha256:69bd1a15d7ba3694631e00df8de65a8cb031911ca11f44929c97fe05eb9b6c1d", + "sha256:6bf088c1ce160f50ea40764f825ec9b72ed9da25346216b91361eef8ad1b8f8c", + "sha256:6e8c66f70fb539301e064f6478d7453e820d8a2c631da948a23384865cd95544", + "sha256:727dd1389bc5cb9827cbd1f9d40d2c2a1a0c9b32dd2261db522d22a604a6eec9", + "sha256:74a04183e6e64930b667d321524e3c5361094bb4af9083db5c301db64cd341f3", + "sha256:75e636fd3e0fb872693f23ccb8a5ff2cd578801251f3a4f6854c6a5d437d3c04", + "sha256:7761afe0126d046974a01e030ae7529ed0ca6a196de3ec6937c11df0df1bc91c", + "sha256:7888310f6214f19ab2b6df90f3f06afa3df7ef7355fc025e78a3044737fab1f5", + "sha256:7b0554af24df2bf96618dac71ddada02420f946be943b181108cac55a7a2dcd4", + "sha256:7c7b502bc34f6e32ba022b4a209638f9e097d7a9098104ae420eb8186217ebbb", + "sha256:808add66ea764ed97d44dda1ac4f2cfec4c1867d9efb16a33d158be79f32b8a4", + "sha256:831e648102c82f152e14c1a0938689dbb22480c548c8d4b8b248b3e50967b88c", + "sha256:93689632949aff41199090eff5474f3990b6823404e45d66a5d44304e9cdc467", + "sha256:96b5e6874431df16aee0c1ba237574cb6dff1dcb173798faa6a9d8b399a05d0e", + "sha256:9a54614049a18a2d6fe156e68e188da02a046a4a93cf24f373bffd977e943421", + "sha256:a138441e95562b3c078746a22f8fca8ff1c22c014f856278bdbdd89ca36cff1b", + "sha256:a647c0d4478b995c5e54615a2e5360ccedd2f85e70ab57fbe817ca613d5e63b8", + "sha256:a9c9bc489f8ab30906d7a85afac4b4944a572a7432e00698a7239f44a44e6efb", + "sha256:ad2277b185ebce47a63f4dc6302e30f05762b688f8dc3de55dbae4651872cdf3", + "sha256:b6d5e92df2b77665e07ddb2e4dbd6d644b78e4c0d2e9272a852627cdba0d75cf", + "sha256:bc431b065722a5ad1dfb4df354fb9333b7a582a5ee39a90e6ffff688d72f27a1", + "sha256:bdd0de2d64688ecae88dd8935012c4a72681e5df632af903a1dca8c5e7aa871a", + "sha256:c79698d4cd9318d9481d89a77e2d3fcaeff5486be641e60a4b49f3d2ecca4e28", + "sha256:cb6259196a589123d755380b65127ddc60f4c64b21fc3bb46ce3a6ea663659b0", + "sha256:d5b87da55a08acb586bad5c3aa3b86505f559b84f39035b233d5bf844b0834b1", + "sha256:dcd7b9c7139dc8258d164b55696ecd16c04607f1cc33ba7af86613881ffe4ac8", + "sha256:dfe4c1fedfde4e2fbc009d5ad420647f7730d719786388b7de0999bf32c0d9fd", + "sha256:ea98f633d45f7e815db648fd7ff0f19e328302ac36427343e4432c84432e7ff4", + "sha256:ec52c351b35ca269cb1f8069d610fc45c5bd38c3e91f9ab4cbbf0aebc136d9c8", + "sha256:eef7592281f7c174d3d6cbfbb7ee5984a671fcd77e3fc78e973d492e9bf0eb3f", + "sha256:f07f1f00e22b231dd3d9b9208692042e29792d6bd4f6639415d2f23158a80013", + "sha256:f3fac744f9b540148fa7715a435d2283b71f68bfb6d4aae24482a890aed18b59", + "sha256:fa768eff5f9f958270b081bb33581b4b569faabf8774726b283edb06617101dc", + "sha256:fac2d65901fb0fdf20363fbd345c01958a742f2dc62a8dd4495af66e3ff502a4" ], "markers": "python_version >= '3.7'", - "version": "==9.1.1" + "version": "==9.2.0" + }, + "prometheus-client": { + "hashes": [ + "sha256:522fded625282822a89e2773452f42df14b5a8e84a86433e3f8a189c1d54dc01", + "sha256:5459c427624961076277fdc6dc50540e2bacb98eebde99886e59ec55ed92093a" + ], + "markers": "python_version >= '3.6'", + "version": "==0.14.1" }, "prompt-toolkit": { "hashes": [ @@ -711,6 +793,12 @@ "markers": "python_full_version >= '3.6.8'", "version": "==3.0.9" }, + "python-crontab": { + "hashes": [ + "sha256:1e35ed7a3cdc3100545b43e196d34754e6551e7f95e4caebbe0e1c0ca41c2f1b" + ], + "version": "==2.6.0" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", @@ -798,11 +886,11 @@ }, "setuptools": { "hashes": [ - "sha256:990a4f7861b31532871ab72331e755b5f14efbe52d336ea7f6118144dd478741", - "sha256:c1848f654aea2e3526d17fc3ce6aeaa5e7e24e66e645b5be2171f3f6b4e5a178" + "sha256:16923d366ced322712c71ccb97164d07472abeecd13f3a6c283f6d5d26722793", + "sha256:db3b8e2f922b2a910a29804776c643ea609badb6a32c4bcc226fd4fd902cce65" ], "markers": "python_version >= '3.7'", - "version": "==62.6.0" + "version": "==63.1.0" }, "six": { "hashes": [ @@ -832,9 +920,26 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '3.7'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, + "tornado": { + "hashes": [ + "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca", + "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72", + "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23", + "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8", + "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b", + "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9", + "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13", + "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75", + "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac", + "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", + "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" + ], + "markers": "python_version >= '3.7'", + "version": "==6.2" + }, "traitlets": { "hashes": [ "sha256:0bb9f1f9f017aa8ec187d8b1b2a7a6626a2a1d877116baba52a129bfa124f8e2", @@ -843,6 +948,14 @@ "markers": "python_version >= '3.7'", "version": "==5.3.0" }, + "tzdata": { + "hashes": [ + "sha256:238e70234214138ed7b4e8a0fab0e5e13872edab3be586ab8198c407620e2ab9", + "sha256:8b536a8ec63dc0751342b3984193a3118f8fca2afe25752bb9b7fffd398552d3" + ], + "markers": "python_version >= '2'", + "version": "==2022.1" + }, "uritemplate": { "hashes": [ "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", @@ -853,11 +966,11 @@ }, "urllib3": { "hashes": [ - "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", - "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", + "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.9" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.10" }, "vine": { "hashes": [ @@ -964,11 +1077,11 @@ }, "awscli": { "hashes": [ - "sha256:603e259e16a790ce659c5d9aee25439d2e2f2bce3614c218a63b34e85d0afdec", - "sha256:8e814695a02cbbc0515160ec000336f3586d04e698e7c8066233fad3021ef051" + "sha256:74a30fa42cf381faba9a4ce4a5d222d729c7294dd460d79641802218ff0ceefd", + "sha256:dd7da9d425274d6dbcf3b6d8e4b0aca266e97e2fe979f871369b42cbf2c1f7a3" ], "index": "pypi", - "version": "==1.25.20" + "version": "==1.25.26" }, "awscli-local": { "hashes": [ @@ -1077,11 +1190,11 @@ }, "faker": { "hashes": [ - "sha256:0297b7fc0f2458dfff8d5a92335c62fa25fb059f8cbaf7db580a0dd7177aff2e", - "sha256:b9f93ec97a70da79d43f497aa7b2b7d2bcd5d0c6d3ab7c102dde4193d0a38351" + "sha256:8e94a749d2f3d9b367f61eb33be6a534f0a2d305c54e912ee6618370e3278db7", + "sha256:a126fa66f54e65a67f913dcc698c9d023def7277882536bde2968fcac701bfd5" ], "markers": "python_version >= '3.6'", - "version": "==13.14.0" + "version": "==13.15.0" }, "flake8": { "hashes": [ @@ -1351,11 +1464,11 @@ }, "pytest-mock": { "hashes": [ - "sha256:2c6d756d5d3bf98e2e80797a959ca7f81f479e7d1f5f571611b0fdd6d1745240", - "sha256:d989f11ca4a84479e288b0cd1e6769d6ad0d3d7743dcc75e460d1416a5f2135a" + "sha256:77f03f4554392558700295e05aed0b1096a20d4a60a4f3ddcde58b0c31c8fca2", + "sha256:8a9e226d6c0ef09fcf20c94eb3405c388af438a90f3e39687f84166da82d5948" ], "index": "pypi", - "version": "==3.8.1" + "version": "==3.8.2" }, "python-dateutil": { "hashes": [ @@ -1413,7 +1526,7 @@ "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], - "markers": "python_version >= '3.5' and python_version < '4.0'", + "markers": "python_version >= '3.5' and python_version < '4'", "version": "==4.7.2" }, "s3transfer": { @@ -1448,19 +1561,19 @@ }, "typing-extensions": { "hashes": [ - "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708", - "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376" + "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", + "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6" ], "markers": "python_version >= '3.7'", - "version": "==4.2.0" + "version": "==4.3.0" }, "urllib3": { "hashes": [ - "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", - "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", + "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.9" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.10" }, "watchdog": { "hashes": [ diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 8547b656db..cac7dd2c58 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -76,6 +76,7 @@ services: ./gunicorn_start.sh" ports: - "8080:8080" + - "5555:5555" depends_on: - clamav-rest - localstack diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index bc8c27d3e9..37ddf511bd 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -47,6 +47,7 @@ class Common(Configuration): "corsheaders", "django_extensions", "drf_yasg", + "django_celery_beat", "storages", # Local apps "tdpservice.core.apps.CoreConfig", From 06df91793b42f627092d03cac37d182da22dda3f Mon Sep 17 00:00:00 2001 From: msohanifr Date: Thu, 14 Jul 2022 11:11:11 -0400 Subject: [PATCH 012/113] added the task for upload --- tdrs-backend/tdpservice/scheduling/tasks.py | 32 +++++++++++++++++++-- tdrs-backend/tdpservice/settings/common.py | 13 +++++++++ tdrs-backend/tdpservice/settings/local.py | 9 ------ 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 189e36ddcc..818a7105b4 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,6 +1,34 @@ from __future__ import absolute_import from celery import shared_task +from django.conf import settings + +import paramiko + + +SERVER_ADDRESS = settings.SERVER_ADDRESS +LOCAL_KEY = settings.LOCAL_KEY +USERNAME = settings.USERNAME + @shared_task -def upload(a,b): - return a*b +def upload(server_address: SERVER_ADDRESS, + local_key: LOCAL_KEY, + username: USERNAME, + source, # includes the file name + destination # includes the file name + ): + transport = paramiko.SSHClient() + transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + transport.connect(server_address, key_filename=local_key, username=username) + + # Upload file + try: + # sftp = transport.open_sftp() + sftp = paramiko.SFTPClient.from_transport(transport) + sftp.put(source, destination) + return True + except Exception as e: + print(e) + return False + + diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 37ddf511bd..2f6a1db677 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -398,3 +398,16 @@ class Common(Configuration): 'AMS_CLIENT_SECRET', '' ) + + # sftp config + SERVER_ADDRESS = '192.168.1.130' + LOCAL_KEY = '/Users/msohani/.ssh/id_rsa_sftp' + USERNAME = 'mat' + + # CELERY CONFIG + CELERY_BROKER_URL = "redis://redis-server:6379" + CELERY_RESULT_BACKEND = "redis://redis-server:6379" + CELERY_ACCEPT_CONTENT = ['application/json'] + CELERY_TASK_SERIALIZER = 'json' + CELERY_RESULT_SERIALIZER = 'json' + CELERY_TIMEZONE = 'UTC' diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 2b3008e022..83be95bd3b 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -43,12 +43,3 @@ class Local(Common): 'level': 'DEBUG', 'handlers': ['console'] } - - # CELERY CONFIG - CELERY_BROKER_URL = "redis://redis-server:6379" - CELERY_RESULT_BACKEND = "redis://redis-server:6379" - CELERY_ACCEPT_CONTENT = ['application/json'] - CELERY_TASK_SERIALIZER = 'json' - CELERY_RESULT_SERIALIZER = 'json' - CELERY_TIMEZONE = 'UTC' - From 06d2f0d13e638bec12a4bd80cdf73134d3e0b581 Mon Sep 17 00:00:00 2001 From: msohanifr Date: Thu, 14 Jul 2022 19:22:27 -0400 Subject: [PATCH 013/113] added paramiko --- tdrs-backend/Pipfile | 1 + tdrs-backend/Pipfile.lock | 153 ++++++++++++++++++++++++-------------- 2 files changed, 98 insertions(+), 56 deletions(-) diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index c52216e805..e50d2967f9 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -49,6 +49,7 @@ celery = "*" redis = "*" flower = "1.1.0" django-celery-beat = "2.3.0" +paramiko = "2.11.0" [requires] python_version = "3.10.4" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index 16b5b997fa..c0b4efa4bb 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "50b95aa34c5149af5d8d39f33b0b20fa505925e63e641fae04d04d81e9d1b38c" + "sha256": "48ccbab3ee5d988f00d302c10a62be2e8261eb3d617dcb31c39c0222436abcc7" }, "pipfile-spec": 6, "requires": { @@ -62,6 +62,23 @@ ], "version": "==0.2.0" }, + "bcrypt": { + "hashes": [ + "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521", + "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb", + "sha256:4e029cef560967fb0cf4a802bcf4d562d3d6b4b1bf81de5ec1abbe0f1adb027e", + "sha256:61bae49580dce88095d669226d5076d0b9d927754cedbdf76c6c9f5099ad6f26", + "sha256:6d2cb9d969bfca5bc08e45864137276e4c3d3d7de2b162171def3d188bf9d34a", + "sha256:7180d98a96f00b1050e93f5b0f556e658605dd9f524d0b0e68ae7944673f525e", + "sha256:7d9ba2e41e330d2af4af6b1b6ec9e6128e91343d0b4afb9282e54e5508f31baa", + "sha256:7ff2069240c6bbe49109fe84ca80508773a904f5a8cb960e02a977f7f519b129", + "sha256:88273d806ab3a50d06bc6a2fc7c87d737dd669b76ad955f449c43095389bc8fb", + "sha256:a2c46100e315c3a5b90fdc53e429c006c5f962529bc27e1dfd656292c20ccc40", + "sha256:cd43303d6b8a165c29ec6756afd169faba9396a9472cdff753fe9f19b96ce2fa" + ], + "markers": "python_version >= '3.6'", + "version": "==3.2.2" + }, "billiard": { "hashes": [ "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", @@ -191,7 +208,7 @@ "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" ], - "markers": "python_version < '4' and python_full_version >= '3.6.2'", + "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'", "version": "==0.3.0" }, "click-plugins": { @@ -378,7 +395,7 @@ "sha256:15746ed367a5a32eda76cfa2886eeec1de8cda79f519b7c5e12f87ed7cdbd663", "sha256:199f211082eeac7e83563929b8ce41399c1c0f00dfc2f36bc00bea381027eaaa" ], - "markers": "python_version >= '3.7' and python_version < '4'", + "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", "version": "==5.0" }, "django-unique-upload": { @@ -583,6 +600,14 @@ "markers": "python_version >= '3.6'", "version": "==21.3" }, + "paramiko": { + "hashes": [ + "sha256:003e6bee7c034c21fbb051bf83dc0a9ee4106204dd3c53054c71452cc4ec3938", + "sha256:655f25dc8baf763277b933dfcea101d636581df8d6b9774d1fb653426b72c270" + ], + "index": "pypi", + "version": "==2.11.0" + }, "parso": { "hashes": [ "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0", @@ -785,6 +810,22 @@ "index": "pypi", "version": "==2.4.0" }, + "pynacl": { + "hashes": [ + "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", + "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", + "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", + "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", + "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", + "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", + "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", + "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", + "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", + "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543" + ], + "markers": "python_version >= '3.6'", + "version": "==1.5.0" + }, "pyparsing": { "hashes": [ "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", @@ -886,11 +927,11 @@ }, "setuptools": { "hashes": [ - "sha256:16923d366ced322712c71ccb97164d07472abeecd13f3a6c283f6d5d26722793", - "sha256:db3b8e2f922b2a910a29804776c643ea609badb6a32c4bcc226fd4fd902cce65" + "sha256:0d33c374d41c7863419fc8f6c10bfe25b7b498aa34164d135c622e52580c6b16", + "sha256:c04b44a57a6265fe34a4a444e965884716d34bae963119a76353434d6f18e450" ], "markers": "python_version >= '3.7'", - "version": "==63.1.0" + "version": "==63.2.0" }, "six": { "hashes": [ @@ -969,7 +1010,7 @@ "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", "version": "==1.26.10" }, "vine": { @@ -1077,11 +1118,11 @@ }, "awscli": { "hashes": [ - "sha256:74a30fa42cf381faba9a4ce4a5d222d729c7294dd460d79641802218ff0ceefd", - "sha256:dd7da9d425274d6dbcf3b6d8e4b0aca266e97e2fe979f871369b42cbf2c1f7a3" + "sha256:671e735d7efb052092f38dc453398c37cbfe191ee4ecbf18cfd6d3f98e8ad629", + "sha256:9a049a0bde495954c14676ec2750a1acaf00240e4bc53282acc6db0a40ee748b" ], "index": "pypi", - "version": "==1.25.26" + "version": "==1.25.30" }, "awscli-local": { "hashes": [ @@ -1127,50 +1168,50 @@ "toml" ], "hashes": [ - "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749", - "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982", - "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3", - "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9", - "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428", - "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e", - "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c", - "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9", - "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264", - "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605", - "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397", - "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d", - "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c", - "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815", - "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068", - "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b", - "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4", - "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4", - "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3", - "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84", - "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83", - "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4", - "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8", - "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb", - "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d", - "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df", - "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6", - "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b", - "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72", - "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13", - "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df", - "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc", - "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6", - "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28", - "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b", - "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4", - "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad", - "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46", - "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3", - "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9", - "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54" + "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32", + "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7", + "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996", + "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55", + "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46", + "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de", + "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039", + "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee", + "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1", + "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f", + "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63", + "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083", + "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe", + "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0", + "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6", + "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe", + "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933", + "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0", + "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c", + "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07", + "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8", + "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b", + "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e", + "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120", + "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f", + "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e", + "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd", + "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f", + "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386", + "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8", + "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae", + "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc", + "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783", + "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d", + "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c", + "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97", + "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978", + "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf", + "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29", + "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39", + "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452" ], "markers": "python_version >= '3.7'", - "version": "==6.4.1" + "version": "==6.4.2" }, "docutils": { "hashes": [ @@ -1572,7 +1613,7 @@ "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", "version": "==1.26.10" }, "watchdog": { @@ -1608,11 +1649,11 @@ }, "zipp": { "hashes": [ - "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad", - "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099" + "sha256:05b45f1ee8f807d0cc928485ca40a07cb491cf092ff587c0df9cb1fd154848d2", + "sha256:47c40d7fe183a6f21403a199b3e4192cca5774656965b0a4988ad2f8feb5f009" ], "markers": "python_version >= '3.7'", - "version": "==3.8.0" + "version": "==3.8.1" } } } From 15daafb4a6c1f3c4e07c16ded10066f3298512ec Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 19 Jul 2022 06:38:09 -0400 Subject: [PATCH 014/113] added upload task --- tdrs-backend/docker-compose.yml | 2 +- .../tdpservice/data_files/serializers.py | 6 ++++ tdrs-backend/tdpservice/data_files/views.py | 6 +--- tdrs-backend/tdpservice/scheduling/tasks.py | 36 ++++++++++++++++--- tdrs-backend/tdpservice/settings/common.py | 6 ++-- 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index cac7dd2c58..3d1307d35b 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -73,7 +73,7 @@ services: build: . command: > bash -c "./wait_for_services.sh && - ./gunicorn_start.sh" + ./gunicorn_start.sh && celery -A tdpservice.settings worker -l info" ports: - "8080:8080" - "5555:5555" diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index 44d7467868..ceed379c92 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -11,6 +11,7 @@ from tdpservice.security.models import ClamAVFileScan from tdpservice.stts.models import STT from tdpservice.users.models import User +from tdpservice.scheduling import tasks class DataFileSerializer(serializers.ModelSerializer): @@ -54,6 +55,11 @@ def create(self, validated_data): av_scan.save() # TODO: Add Titan upload here + print(data_file.__dict__) + tasks.upload.delay( + source=str(data_file.file), + destination=str(data_file.file).split('/')[-1] + ) return data_file diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index f3c95f88e6..51c127b7c1 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -39,7 +39,7 @@ class DataFileViewSet(ModelViewSet): http_method_names = ['get', 'post', 'head'] filterset_class = DataFileFilter parser_classes = [MultiPartParser] - permission_classes = [DataFilePermissions] + #permission_classes = [DataFilePermissions] serializer_class = DataFileSerializer # TODO: Handle versioning in queryset @@ -51,10 +51,6 @@ class DataFileViewSet(ModelViewSet): # we will be able to appropriately refer to the latest versions only. ordering = ['-version'] - def perform_create(self): - """ - TODO: create has to be done here - """ def filter_queryset(self, queryset): """Only apply filters to the list action.""" diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 818a7105b4..39599d77c6 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -4,19 +4,44 @@ import paramiko - SERVER_ADDRESS = settings.SERVER_ADDRESS LOCAL_KEY = settings.LOCAL_KEY USERNAME = settings.USERNAME +''' + +from django.utils import timezone +last_24 = timezone.now()-timezone.timedelta(hours=24) +DataFile.objects.filter(created_at__gt=last_24) + +# what should the filename be? +with open('write.txt','wb') as f1: + ...: with df.file.file.open() as f2: + ...: f1.write(f2.read()) + + +# ssh connect and command +_stdin, _stdout,_stderr = transport.exec_command("ls") +print(_stdout.read().decode()) + +1) try to create directory +if successful, then good, +if not, then either directory exists, or something else happened. + in case of error, we can list the directories, and confirm if it exists, or we can do this first + +2) Query DISTINCT values based on year, STT + +''' + @shared_task -def upload(server_address: SERVER_ADDRESS, - local_key: LOCAL_KEY, - username: USERNAME, +def upload( source, # includes the file name destination # includes the file name ): + server_address = SERVER_ADDRESS + local_key = LOCAL_KEY + username = USERNAME transport = paramiko.SSHClient() transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) transport.connect(server_address, key_filename=local_key, username=username) @@ -24,7 +49,7 @@ def upload(server_address: SERVER_ADDRESS, # Upload file try: # sftp = transport.open_sftp() - sftp = paramiko.SFTPClient.from_transport(transport) + sftp = transport.open_sftp() sftp.put(source, destination) return True except Exception as e: @@ -32,3 +57,4 @@ def upload(server_address: SERVER_ADDRESS, return False + diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 2f6a1db677..c5af830f56 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -47,7 +47,7 @@ class Common(Configuration): "corsheaders", "django_extensions", "drf_yasg", - "django_celery_beat", + #"django_celery_beat", "storages", # Local apps "tdpservice.core.apps.CoreConfig", @@ -55,7 +55,7 @@ class Common(Configuration): "tdpservice.stts", "tdpservice.data_files", "tdpservice.security", - "tdpservice.scheduling", + "tdpservice.scheduling" ) # https://docs.djangoproject.com/en/2.0/topics/http/middleware/ @@ -401,7 +401,7 @@ class Common(Configuration): # sftp config SERVER_ADDRESS = '192.168.1.130' - LOCAL_KEY = '/Users/msohani/.ssh/id_rsa_sftp' + LOCAL_KEY = '/tdpapp/id_rsa_sftp' USERNAME = 'mat' # CELERY CONFIG From 661c411848ac0d4439aa15aa37c29f1daf5383e4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 19 Jul 2022 12:15:53 -0400 Subject: [PATCH 015/113] added shceduled tasks --- tdrs-backend/tdpservice/scheduling/tasks.py | 6 ++++-- tdrs-backend/tdpservice/settings/celery.py | 21 +++++++++++++++++++++ tdrs-backend/tdpservice/settings/common.py | 2 +- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 39599d77c6..6347474b7f 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -56,5 +56,7 @@ def upload( print(e) return False - - +@shared_task +def run_backup(b): + """ No params, setup for actual backup call. """ + logger.debug("my arg was" + b) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index d3245705f5..b01ff489a2 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -23,3 +23,24 @@ @app.task(bind=True) def debug_task(self): print('Request: {0!r}'.format(self.request)) + +import logging +logger = logging.getLogger(__name__) +from celery.schedules import crontab +from tdpservice.scheduling.tasks import run_backup + +@app.on_after_configure.connect +def setup_periodic_tasks(sender, **kwargs): + + # Executes every Monday morning at 7:30 a.m. + sender.add_periodic_task( + crontab(hour='*', minute='*/1', day_of_week='*'), + run_backup.s('Happy Mondays!'), + ) + +""" +@app.task +def run_backup(b): + + logger.debug("my arg was" + b) +""" diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index c5af830f56..8fc2845e45 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -47,7 +47,7 @@ class Common(Configuration): "corsheaders", "django_extensions", "drf_yasg", - #"django_celery_beat", + "django_celery_beat", "storages", # Local apps "tdpservice.core.apps.CoreConfig", From c0a2652ba0450e6a092a8fba89b7286cb06013a4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 21 Jul 2022 10:55:23 -0400 Subject: [PATCH 016/113] Working scheduler --- tdrs-backend/gunicorn_start.sh | 9 ++++++++- tdrs-backend/tdpservice/scheduling/tasks.py | 5 +++++ tdrs-backend/tdpservice/settings/celery.py | 3 +-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index ddc6921201..9b7f67de34 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -6,6 +6,13 @@ python manage.py makemigrations python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput + +# +celery -A tdpservice.settings worker -l info & +sleep 5 +celery -A tdpservice.settings --broker=redis://redis-server:6379 flower & +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & + echo "Starting Gunicorn" if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then gunicorn_params="--bind 0.0.0.0:8080 --timeout 10 --workers 3 --reload --log-level $LOGGING_LEVEL" @@ -15,4 +22,4 @@ fi gunicorn_cmd="gunicorn tdpservice.wsgi:application $gunicorn_params" -exec $gunicorn_cmd \ No newline at end of file +exec $gunicorn_cmd diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 6347474b7f..a926ee6b6e 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -31,6 +31,8 @@ 2) Query DISTINCT values based on year, STT +NEW: Do not need this anymore. Should schedule an upload to the server as soon as the file is uploaded. +However, we will need the file information, to know where it is being uploaded. ''' @@ -56,6 +58,9 @@ def upload( print(e) return False +import logging +logger = logging.getLogger(__name__) + @shared_task def run_backup(b): """ No params, setup for actual backup call. """ diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index b01ff489a2..ca6c7cf3ca 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -24,8 +24,7 @@ def debug_task(self): print('Request: {0!r}'.format(self.request)) -import logging -logger = logging.getLogger(__name__) + from celery.schedules import crontab from tdpservice.scheduling.tasks import run_backup From 2f30262b6005ed7c881a76e0e26bb693dc39ccf4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Sun, 24 Jul 2022 22:54:30 -0400 Subject: [PATCH 017/113] FileUpload --- .../tdpservice/data_files/serializers.py | 8 --- tdrs-backend/tdpservice/data_files/views.py | 13 ++++ tdrs-backend/tdpservice/scheduling/tasks.py | 65 ++++++++++--------- tdrs-backend/tdpservice/settings/celery.py | 2 +- 4 files changed, 48 insertions(+), 40 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index ceed379c92..c8492e05cb 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -11,7 +11,6 @@ from tdpservice.security.models import ClamAVFileScan from tdpservice.stts.models import STT from tdpservice.users.models import User -from tdpservice.scheduling import tasks class DataFileSerializer(serializers.ModelSerializer): @@ -54,13 +53,6 @@ def create(self, validated_data): av_scan.data_file = data_file av_scan.save() - # TODO: Add Titan upload here - print(data_file.__dict__) - tasks.upload.delay( - source=str(data_file.file), - destination=str(data_file.file).split('/')[-1] - ) - return data_file def update(self, instance, validated_data): diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 51c127b7c1..ada696fe4b 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -33,6 +33,8 @@ class Meta: fields = ['stt', 'quarter', 'year'] +from tdpservice.scheduling import tasks + class DataFileViewSet(ModelViewSet): """Data file views.""" @@ -51,6 +53,17 @@ class DataFileViewSet(ModelViewSet): # we will be able to appropriately refer to the latest versions only. ordering = ['-version'] + def create(self, request, *args, **kwargs): + response = super().create(request, *args, **kwargs) + data_file = DataFile.objects.get(id=response.data.get('id')) + print('----------------------:',str(data_file.file).split('/')[-1]) + tasks.run_backup.delay('======================') + tasks.upload.delay( + source=str(data_file.file), + destination=str(data_file.file).split('/')[-1], + quarter=data_file.quarter + ) + return response def filter_queryset(self, queryset): """Only apply filters to the list action.""" diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index a926ee6b6e..374dddf3a6 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,67 +1,70 @@ from __future__ import absolute_import from celery import shared_task from django.conf import settings - +import datetime import paramiko SERVER_ADDRESS = settings.SERVER_ADDRESS LOCAL_KEY = settings.LOCAL_KEY USERNAME = settings.USERNAME -''' - -from django.utils import timezone -last_24 = timezone.now()-timezone.timedelta(hours=24) -DataFile.objects.filter(created_at__gt=last_24) - -# what should the filename be? -with open('write.txt','wb') as f1: - ...: with df.file.file.open() as f2: - ...: f1.write(f2.read()) - - -# ssh connect and command -_stdin, _stdout,_stderr = transport.exec_command("ls") -print(_stdout.read().decode()) - -1) try to create directory -if successful, then good, -if not, then either directory exists, or something else happened. - in case of error, we can list the directories, and confirm if it exists, or we can do this first - -2) Query DISTINCT values based on year, STT - -NEW: Do not need this anymore. Should schedule an upload to the server as soon as the file is uploaded. -However, we will need the file information, to know where it is being uploaded. -''' - +import logging +logger = logging.getLogger(__name__) @shared_task def upload( - source, # includes the file name - destination # includes the file name + source, # the file name + destination, # the file name + quarter, # Quarter as number in (1,2,3,4) ): + print('-------------------------in upload 1') + today_date = datetime.datetime.today() + upper_directory_name = today_date.strftime('%Y%m%d') + lower_directory_name = today_date.strftime('%Y-' + str(quarter)) server_address = SERVER_ADDRESS local_key = LOCAL_KEY username = USERNAME + logger.debug('in upload 2') transport = paramiko.SSHClient() + logger.debug('in upload 3') transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + logger.debug('in upload 4') transport.connect(server_address, key_filename=local_key, username=username) + sftp = transport.open_sftp() + + # Check if UPPER directory exists, then change to that directory + try: + sftp.chdir(upper_directory_name) + except IOError as e: + sftp.mkdir(upper_directory_name) + sftp.chdir(upper_directory_name) + + # Check if LOWER directory exists + try: + sftp.chdir(lower_directory_name) + except IOError as e: + sftp.mkdir(lower_directory_name) + sftp.chdir(lower_directory_name) + + logger.debug('++++++++++++++++ in run upload') + # Upload file try: # sftp = transport.open_sftp() - sftp = transport.open_sftp() sftp.put(source, destination) return True except Exception as e: print(e) return False + import logging logger = logging.getLogger(__name__) + @shared_task def run_backup(b): """ No params, setup for actual backup call. """ logger.debug("my arg was" + b) + print('00000000000000000000000000000000000000000') diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index ca6c7cf3ca..5bd86f7ccf 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -33,7 +33,7 @@ def setup_periodic_tasks(sender, **kwargs): # Executes every Monday morning at 7:30 a.m. sender.add_periodic_task( - crontab(hour='*', minute='*/1', day_of_week='*'), + crontab(hour='*', minute='1', day_of_week='*'), run_backup.s('Happy Mondays!'), ) From c587b5940b710fb40f4f6bc325c84b0f22153aaf Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 25 Jul 2022 00:59:33 -0400 Subject: [PATCH 018/113] FileUpload - Refactored scheduled task to let delay task to be scheduled --- tdrs-backend/tdpservice/data_files/views.py | 3 ++- tdrs-backend/tdpservice/scheduling/tasks.py | 17 ++++++++++++++++- tdrs-backend/tdpservice/settings/celery.py | 12 +++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index ada696fe4b..dddfa826d7 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -61,7 +61,8 @@ def create(self, request, *args, **kwargs): tasks.upload.delay( source=str(data_file.file), destination=str(data_file.file).split('/')[-1], - quarter=data_file.quarter + quarter=data_file.quarter, + data_file_pk=response.data.get('id') ) return response diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 374dddf3a6..346c883319 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,9 +1,14 @@ from __future__ import absolute_import + +import os + from celery import shared_task from django.conf import settings import datetime import paramiko +from tdpservice.data_files.models import DataFile + SERVER_ADDRESS = settings.SERVER_ADDRESS LOCAL_KEY = settings.LOCAL_KEY USERNAME = settings.USERNAME @@ -16,6 +21,7 @@ def upload( source, # the file name destination, # the file name quarter, # Quarter as number in (1,2,3,4) + data_file_pk, ): print('-------------------------in upload 1') today_date = datetime.datetime.today() @@ -33,6 +39,13 @@ def upload( sftp = transport.open_sftp() + data_file = DataFile.objects.get(id=data_file_pk) + print('________________________________________________') + f = data_file.file.read() + with open('temp_file', 'wb') as f1: + f1.write(f) + f1.close() + print('______________________GG________________________') # Check if UPPER directory exists, then change to that directory try: sftp.chdir(upper_directory_name) @@ -52,7 +65,8 @@ def upload( # Upload file try: # sftp = transport.open_sftp() - sftp.put(source, destination) + sftp.put('temp_file', destination) + os.remove('temp_file') return True except Exception as e: print(e) @@ -68,3 +82,4 @@ def run_backup(b): """ No params, setup for actual backup call. """ logger.debug("my arg was" + b) print('00000000000000000000000000000000000000000') + return 0 diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 5bd86f7ccf..f97e55ffaf 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -26,8 +26,16 @@ def debug_task(self): from celery.schedules import crontab -from tdpservice.scheduling.tasks import run_backup +app.conf.beat_schedule = { + 'example': { + 'task': 'tdpservice.scheduling.tasks.run_backup', + 'schedule': crontab(minute='10'), + 'args': ['lolo'], + }, +} + +""" @app.on_after_configure.connect def setup_periodic_tasks(sender, **kwargs): @@ -36,6 +44,8 @@ def setup_periodic_tasks(sender, **kwargs): crontab(hour='*', minute='1', day_of_week='*'), run_backup.s('Happy Mondays!'), ) +""" + """ @app.task From 5a2b3b5949db9f37715def597b32f4001d825d19 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 25 Jul 2022 11:56:34 -0400 Subject: [PATCH 019/113] FileUpload - added error handling --- tdrs-backend/tdpservice/data_files/views.py | 2 -- tdrs-backend/tdpservice/scheduling/tasks.py | 23 ++------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index dddfa826d7..9511fbc570 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -56,8 +56,6 @@ class DataFileViewSet(ModelViewSet): def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) data_file = DataFile.objects.get(id=response.data.get('id')) - print('----------------------:',str(data_file.file).split('/')[-1]) - tasks.run_backup.delay('======================') tasks.upload.delay( source=str(data_file.file), destination=str(data_file.file).split('/')[-1], diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 346c883319..96eae27946 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -23,29 +23,23 @@ def upload( quarter, # Quarter as number in (1,2,3,4) data_file_pk, ): - print('-------------------------in upload 1') today_date = datetime.datetime.today() upper_directory_name = today_date.strftime('%Y%m%d') lower_directory_name = today_date.strftime('%Y-' + str(quarter)) server_address = SERVER_ADDRESS local_key = LOCAL_KEY username = USERNAME - logger.debug('in upload 2') transport = paramiko.SSHClient() - logger.debug('in upload 3') transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - logger.debug('in upload 4') transport.connect(server_address, key_filename=local_key, username=username) sftp = transport.open_sftp() data_file = DataFile.objects.get(id=data_file_pk) - print('________________________________________________') f = data_file.file.read() with open('temp_file', 'wb') as f1: f1.write(f) f1.close() - print('______________________GG________________________') # Check if UPPER directory exists, then change to that directory try: sftp.chdir(upper_directory_name) @@ -60,26 +54,13 @@ def upload( sftp.mkdir(lower_directory_name) sftp.chdir(lower_directory_name) - logger.debug('++++++++++++++++ in run upload') - # Upload file try: # sftp = transport.open_sftp() sftp.put('temp_file', destination) os.remove('temp_file') + logger.info('File {} has been successfully uploaded to {}'.format(destination, server_address)) return True except Exception as e: - print(e) + logger.error('Failed to upload {} with error:{}'.format(destination, e)) return False - - -import logging -logger = logging.getLogger(__name__) - - -@shared_task -def run_backup(b): - """ No params, setup for actual backup call. """ - logger.debug("my arg was" + b) - print('00000000000000000000000000000000000000000') - return 0 From 628bc2ac18e462718a17c60f77bef4de9d5dbea4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 28 Jul 2022 10:21:09 -0400 Subject: [PATCH 020/113] 1133 - Added env vars, creating directories, putting the file, error handling. --- tdrs-backend/docker-compose.local.yml | 2 + tdrs-backend/docker-compose.yml | 3 + .../tdpservice/data_files/test/test_models.py | 2 +- tdrs-backend/tdpservice/data_files/views.py | 16 ++--- .../tdpservice/scheduling/__init__.py | 28 ++++++++ tdrs-backend/tdpservice/scheduling/tasks.py | 69 +++++++++---------- .../tdpservice/scheduling/test/__init__.py | 0 .../scheduling/test/test_file_upload.py | 30 ++++++++ tdrs-backend/tdpservice/security/__init__.py | 1 + tdrs-backend/tdpservice/settings/celery.py | 25 +------ tdrs-backend/tdpservice/settings/common.py | 6 +- tdrs-backend/tdpservice/stts/test/test_api.py | 2 +- .../tdpservice/stts/test/test_commands.py | 2 +- .../tdpservice/stts/test/test_models.py | 2 +- .../tdpservice/users/test/test_permissions.py | 21 +++++- 15 files changed, 135 insertions(+), 74 deletions(-) create mode 100644 tdrs-backend/tdpservice/scheduling/test/__init__.py create mode 100644 tdrs-backend/tdpservice/scheduling/test/test_file_upload.py diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml index 518dd07982..c81b391243 100644 --- a/tdrs-backend/docker-compose.local.yml +++ b/tdrs-backend/docker-compose.local.yml @@ -22,6 +22,8 @@ services: - DATA_DIR=/tmp/localstack/data - AWS_BUCKET=tdp-datafiles-localstack - AWS_REGION_NAME=us-gov-west-1 + - HOSTNAME=localstack + - HOSTNAME_EXTERNAL=localstack ports: - "4566:4566" volumes: diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 3d1307d35b..9d9a448a23 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -67,6 +67,9 @@ services: - AMS_CLIENT_ID - AMS_CLIENT_SECRET - AMS_CONFIGURATION_ENDPOINT + - ACFTITAN_HOST + - ACFTITAN_KEY + - ACFTITAN_USERNAME volumes: - .:/tdpapp image: tdp diff --git a/tdrs-backend/tdpservice/data_files/test/test_models.py b/tdrs-backend/tdpservice/data_files/test/test_models.py index 8dd2de055f..2783759c89 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_models.py +++ b/tdrs-backend/tdpservice/data_files/test/test_models.py @@ -3,7 +3,7 @@ from tdpservice.stts.models import STT -from ..models import DataFile +from tdpservice.data_files.models import DataFile @pytest.mark.django_db diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 9511fbc570..8b3282a0f2 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -12,6 +12,7 @@ from rest_framework.viewsets import ModelViewSet from rest_framework.decorators import action from wsgiref.util import FileWrapper +from rest_framework import status from tdpservice.data_files.serializers import DataFileSerializer from tdpservice.data_files.models import DataFile @@ -41,7 +42,7 @@ class DataFileViewSet(ModelViewSet): http_method_names = ['get', 'post', 'head'] filterset_class = DataFileFilter parser_classes = [MultiPartParser] - #permission_classes = [DataFilePermissions] + permission_classes = [DataFilePermissions] serializer_class = DataFileSerializer # TODO: Handle versioning in queryset @@ -55,13 +56,12 @@ class DataFileViewSet(ModelViewSet): def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) - data_file = DataFile.objects.get(id=response.data.get('id')) - tasks.upload.delay( - source=str(data_file.file), - destination=str(data_file.file).split('/')[-1], - quarter=data_file.quarter, - data_file_pk=response.data.get('id') - ) + + # Upload to ACF-TITAN only if file is passed the virus scan and created + if response.status_code == status.HTTP_201_CREATED: + tasks.upload.delay( + data_file_pk=response.data.get('id') + ) return response def filter_queryset(self, queryset): diff --git a/tdrs-backend/tdpservice/scheduling/__init__.py b/tdrs-backend/tdpservice/scheduling/__init__.py index e69de29bb2..4cda3769a7 100644 --- a/tdrs-backend/tdpservice/scheduling/__init__.py +++ b/tdrs-backend/tdpservice/scheduling/__init__.py @@ -0,0 +1,28 @@ +from django.conf import settings +settings.LOCAL_KEY = """-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAQEAun8bpLADVmGiq9BkGBmcPP79L7vPEeQBfC50YosrKh04i0Djvmrl +2UPG3TJitk4ngWOcH+dz9uEVGDLVauDBQ5/m9x83q3+rP+RWgCUJq0sQ65t5zpuTJniplx +LpalJTWOIj2yIPnZqqxSgRlXZX2uXhSfPDjkXfz0w6uxIzJrG+ak573G/bN+kWFjakI4Zo +MzBZ95qq8Cr3EFkLhbNZzRFmUpHMApWaL8A5cqcSGCuS6q6Hozs+AW/kjPAgwH+6zs14Va +aZTPUwaPWYW+GWn6Fwknex0iJAd2Nm6PwBod3mn4+ogdl+gFDB2qmwI+JVCg3qrap7xQdP +fnxjpXFC7QAAA9gzkzJFM5MyRQAAAAdzc2gtcnNhAAABAQC6fxuksANWYaKr0GQYGZw8/v +0vu88R5AF8LnRiiysqHTiLQOO+auXZQ8bdMmK2TieBY5wf53P24RUYMtVq4MFDn+b3Hzer +f6s/5FaAJQmrSxDrm3nOm5MmeKmXEulqUlNY4iPbIg+dmqrFKBGVdlfa5eFJ88OORd/PTD +q7EjMmsb5qTnvcb9s36RYWNqQjhmgzMFn3mqrwKvcQWQuFs1nNEWZSkcwClZovwDlypxIY +K5LqroejOz4Bb+SM8CDAf7rOzXhVpplM9TBo9Zhb4ZafoXCSd7HSIkB3Y2bo/AGh3eafj6 +iB2X6AUMHaqbAj4lUKDeqtqnvFB09+fGOlcULtAAAAAwEAAQAAAQBX18kYgEOCK4jorsaG +x4FdBaoSEr9NKP0+Fc+CHxUcyBi7FrxvE4DiHleMFtCCGDvpaRa/vFBV59z8wezeY7EEfk +vW+KBLKl5qQM4xm8B7K2dz4o60L3lXNJyE6UaCavaRNis3a0q5rIFlTiNCKDTniUHdiHED +EYyC4uAC/2aSgXZdW/+uythxLVsRmpA3weFrMAfMRCM45Luxj2N8LZ4R213aPikX+dSMHK +hxnxvGL9zDwMRR3v4jqBYqXaLPpZKedyvrcNc3TqG+RhUsuSU/0j5YEk683zTxWmIc4Ze6 +j/PDXFrSRREwZCuzeL4emkDUSUKdh0FJ1vCEDhT+DuptAAAAgAdM+SFIjOjMd4L+7GidVF +VE3f+OL5Z6rPRNmwx4ifibukHdk9wsHp55yfccxr013xvM/EOmnhNqb5uNj/MgN981/ZFb +RJobRG4NuH1HHZMd9F8BE5JtkUiuuhryxEzfwTpsgEJuCtR5BW8v9laKYkaBViejZ/XM2M +bjIKCeOooOAAAAgQDarA/9tVL2YR0VupZfQQdZL+4lO2RtvddjdSKEECvckxSRIKVA/9HY +KJRHwrNIzA8sPuxNBR+9BRp+cw3cgTIQ/Rp+ISF9PFVxTMnM//vcsUTUxziYvaal3DDIpb +f8KcPJ6ECSVq/TYPYQ+wCLoPM49c8TnW8PxtDb3f+fI5WBdwAAAIEA2lT6BkcR9A6FHB+j +Ad2PpLYJqef1ZCQJrhvQRlQfrqhmAie7TdTWi7eH/jUksh+4j4rlobQ22ryTBKMA91npq8 +8lhuJnULmWzl7CBTR8YQ/6WbY58PTadkk79e9w/4WOWp8b0SxMwtZDnESs5x1dfuUyqqfS +rFBHiFOY1MK2F7sAAAAdbXNvaGFuaUBNb3MtTWFjQm9vay1Qcm8ubG9jYWwBAgMEBQY= +-----END OPENSSH PRIVATE KEY-----""" diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 96eae27946..b9f253b9aa 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -6,59 +6,58 @@ from django.conf import settings import datetime import paramiko - +import logging from tdpservice.data_files.models import DataFile +logger = logging.getLogger(__name__) -SERVER_ADDRESS = settings.SERVER_ADDRESS -LOCAL_KEY = settings.LOCAL_KEY -USERNAME = settings.USERNAME +server_address = settings.SERVER_ADDRESS +local_key = settings.LOCAL_KEY +username = settings.USERNAME + + +def write_key_to_file(private_key): + """ + Paramiko require the key in file object format + """ + with open('temp_key_file', 'w') as f: + f.write(private_key) + f.close() + return 'temp_key_file' -import logging -logger = logging.getLogger(__name__) @shared_task -def upload( - source, # the file name - destination, # the file name - quarter, # Quarter as number in (1,2,3,4) - data_file_pk, - ): +def upload(data_file_pk): + """ + This task uploads the file in DataFile object with + pk = data_file_pk + to sftp server as defined in Settings file + """ + data_file = DataFile.objects.get(id=data_file_pk) + destination = str(data_file.file).split('/')[-1] today_date = datetime.datetime.today() upper_directory_name = today_date.strftime('%Y%m%d') - lower_directory_name = today_date.strftime('%Y-' + str(quarter)) - server_address = SERVER_ADDRESS - local_key = LOCAL_KEY - username = USERNAME + lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) + transport = paramiko.SSHClient() transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - transport.connect(server_address, key_filename=local_key, username=username) - + transport.connect(server_address, key_filename=write_key_to_file(local_key), username=username) sftp = transport.open_sftp() - data_file = DataFile.objects.get(id=data_file_pk) + transport.exec_command('mkdir -p ' + upper_directory_name + + '/' + lower_directory_name) + f = data_file.file.read() - with open('temp_file', 'wb') as f1: + with open(destination, 'wb') as f1: f1.write(f) f1.close() - # Check if UPPER directory exists, then change to that directory - try: - sftp.chdir(upper_directory_name) - except IOError as e: - sftp.mkdir(upper_directory_name) - sftp.chdir(upper_directory_name) - - # Check if LOWER directory exists - try: - sftp.chdir(lower_directory_name) - except IOError as e: - sftp.mkdir(lower_directory_name) - sftp.chdir(lower_directory_name) + logger.info(os.listdir()) # Upload file try: # sftp = transport.open_sftp() - sftp.put('temp_file', destination) - os.remove('temp_file') + # temp file can be the file object + sftp.put(destination, upper_directory_name + '/' + lower_directory_name + '/' + destination) + os.remove(destination) logger.info('File {} has been successfully uploaded to {}'.format(destination, server_address)) return True except Exception as e: diff --git a/tdrs-backend/tdpservice/scheduling/test/__init__.py b/tdrs-backend/tdpservice/scheduling/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py new file mode 100644 index 0000000000..e365d9cea3 --- /dev/null +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -0,0 +1,30 @@ +"""Integration test(s) for clamav-rest operations.""" +from os import remove +from requests.sessions import Session +import pytest + +from django.contrib.admin import site as admin_site +from django.core.files.base import File +from rest_framework.status import HTTP_400_BAD_REQUEST +from tdpservice.scheduling.tasks import upload + +""" +WRITE TEST: + +1. CREATE FILE -> clean_file fixture -> DONE +2. UPLOAD TO ACF_TITAN -> +3. SSH TO ACF TITAN AND MAKE SURE THE FILE EXISTS +""" + + +from tdpservice.data_files.test.factories import DataFileFactory + + +@pytest.fixture() +def upload_file(): + return DataFileFactory() + + +@pytest.mark.django_db +def test_upload_server(upload_file): + assert upload_file.original_filename == 'data_file.txt' diff --git a/tdrs-backend/tdpservice/security/__init__.py b/tdrs-backend/tdpservice/security/__init__.py index e69de29bb2..8b13789179 100644 --- a/tdrs-backend/tdpservice/security/__init__.py +++ b/tdrs-backend/tdpservice/security/__init__.py @@ -0,0 +1 @@ + diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index f97e55ffaf..f93205c5a7 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -2,12 +2,13 @@ import os from celery import Celery, shared_task from django.conf import settings +import configurations +from celery.schedules import crontab # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.local") os.environ.setdefault("DJANGO_CONFIGURATION", "Local") -import configurations configurations.setup() app = Celery('settings') @@ -20,13 +21,12 @@ # Load task modules from all registered Django apps. app.autodiscover_tasks() + @app.task(bind=True) def debug_task(self): print('Request: {0!r}'.format(self.request)) -from celery.schedules import crontab - app.conf.beat_schedule = { 'example': { 'task': 'tdpservice.scheduling.tasks.run_backup', @@ -34,22 +34,3 @@ def debug_task(self): 'args': ['lolo'], }, } - -""" -@app.on_after_configure.connect -def setup_periodic_tasks(sender, **kwargs): - - # Executes every Monday morning at 7:30 a.m. - sender.add_periodic_task( - crontab(hour='*', minute='1', day_of_week='*'), - run_backup.s('Happy Mondays!'), - ) -""" - - -""" -@app.task -def run_backup(b): - - logger.debug("my arg was" + b) -""" diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 8fc2845e45..806efea005 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -400,9 +400,9 @@ class Common(Configuration): ) # sftp config - SERVER_ADDRESS = '192.168.1.130' - LOCAL_KEY = '/tdpapp/id_rsa_sftp' - USERNAME = 'mat' + SERVER_ADDRESS = os.getenv('ACFTITAN_HOST', '') + LOCAL_KEY = os.getenv('ACFTITAN_KEY', '') + USERNAME = os.getenv('ACFTITAN_USERNAME', '') # CELERY CONFIG CELERY_BROKER_URL = "redis://redis-server:6379" diff --git a/tdrs-backend/tdpservice/stts/test/test_api.py b/tdrs-backend/tdpservice/stts/test/test_api.py index 2a7bb07230..d9678d894a 100644 --- a/tdrs-backend/tdpservice/stts/test/test_api.py +++ b/tdrs-backend/tdpservice/stts/test/test_api.py @@ -3,7 +3,7 @@ from django.contrib.auth import get_user_model from django.urls import reverse from rest_framework import status -from ...stts.models import STT, Region +from tdpservice.stts.models import STT, Region User = get_user_model() diff --git a/tdrs-backend/tdpservice/stts/test/test_commands.py b/tdrs-backend/tdpservice/stts/test/test_commands.py index 8cde88c4e2..073ef1847a 100644 --- a/tdrs-backend/tdpservice/stts/test/test_commands.py +++ b/tdrs-backend/tdpservice/stts/test/test_commands.py @@ -2,7 +2,7 @@ from django.core.management import call_command import pytest -from ..models import Region, STT +from tdpservice.stts.models import Region, STT @pytest.mark.django_db diff --git a/tdrs-backend/tdpservice/stts/test/test_models.py b/tdrs-backend/tdpservice/stts/test/test_models.py index 283e089e2e..e1ab4c1540 100644 --- a/tdrs-backend/tdpservice/stts/test/test_models.py +++ b/tdrs-backend/tdpservice/stts/test/test_models.py @@ -1,7 +1,7 @@ """Module for testing the stt and region models.""" import pytest -from ..models import STT, Region +from tdpservice.stts.models import STT, Region @pytest.mark.django_db diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 9f6f9bf306..1b546934ea 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -68,8 +68,25 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'users.view_user', 'data_files.view_legacyfiletransfer', 'data_files.add_legacyfiletransfer', - 'data_files.change_legacyfiletransfer' - + 'data_files.change_legacyfiletransfer', + 'django_celery_beat.add_clockedschedule', + 'django_celery_beat.add_crontabschedule', + 'django_celery_beat.add_intervalschedule', + 'django_celery_beat.add_periodictask', + 'django_celery_beat.add_periodictasks', + 'django_celery_beat.add_solarschedule', + 'django_celery_beat.change_clockedschedule', + 'django_celery_beat.change_crontabschedule', + 'django_celery_beat.change_intervalschedule', + 'django_celery_beat.change_periodictask', + 'django_celery_beat.change_periodictasks', + 'django_celery_beat.change_solarschedule', + 'django_celery_beat.view_clockedschedule', + 'django_celery_beat.view_crontabschedule', + 'django_celery_beat.view_intervalschedule', + 'django_celery_beat.view_periodictask', + 'django_celery_beat.view_periodictasks', + 'django_celery_beat.view_solarschedule', } group_permissions = ofa_system_admin.get_group_permissions() assert group_permissions == expected_permissions From 384830f18d7f654c1c4ba8ebf7c5d466272dd99b Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 29 Jul 2022 08:07:54 -0400 Subject: [PATCH 021/113] 1133 - Removed the scheduled tasks --- .../tdpservice/scheduling/__init__.py | 29 +------------------ tdrs-backend/tdpservice/scheduling/tasks.py | 20 +++++++++++-- tdrs-backend/tdpservice/settings/celery.py | 15 ---------- 3 files changed, 18 insertions(+), 46 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/__init__.py b/tdrs-backend/tdpservice/scheduling/__init__.py index 4cda3769a7..8b13789179 100644 --- a/tdrs-backend/tdpservice/scheduling/__init__.py +++ b/tdrs-backend/tdpservice/scheduling/__init__.py @@ -1,28 +1 @@ -from django.conf import settings -settings.LOCAL_KEY = """-----BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn -NhAAAAAwEAAQAAAQEAun8bpLADVmGiq9BkGBmcPP79L7vPEeQBfC50YosrKh04i0Djvmrl -2UPG3TJitk4ngWOcH+dz9uEVGDLVauDBQ5/m9x83q3+rP+RWgCUJq0sQ65t5zpuTJniplx -LpalJTWOIj2yIPnZqqxSgRlXZX2uXhSfPDjkXfz0w6uxIzJrG+ak573G/bN+kWFjakI4Zo -MzBZ95qq8Cr3EFkLhbNZzRFmUpHMApWaL8A5cqcSGCuS6q6Hozs+AW/kjPAgwH+6zs14Va -aZTPUwaPWYW+GWn6Fwknex0iJAd2Nm6PwBod3mn4+ogdl+gFDB2qmwI+JVCg3qrap7xQdP -fnxjpXFC7QAAA9gzkzJFM5MyRQAAAAdzc2gtcnNhAAABAQC6fxuksANWYaKr0GQYGZw8/v -0vu88R5AF8LnRiiysqHTiLQOO+auXZQ8bdMmK2TieBY5wf53P24RUYMtVq4MFDn+b3Hzer -f6s/5FaAJQmrSxDrm3nOm5MmeKmXEulqUlNY4iPbIg+dmqrFKBGVdlfa5eFJ88OORd/PTD -q7EjMmsb5qTnvcb9s36RYWNqQjhmgzMFn3mqrwKvcQWQuFs1nNEWZSkcwClZovwDlypxIY -K5LqroejOz4Bb+SM8CDAf7rOzXhVpplM9TBo9Zhb4ZafoXCSd7HSIkB3Y2bo/AGh3eafj6 -iB2X6AUMHaqbAj4lUKDeqtqnvFB09+fGOlcULtAAAAAwEAAQAAAQBX18kYgEOCK4jorsaG -x4FdBaoSEr9NKP0+Fc+CHxUcyBi7FrxvE4DiHleMFtCCGDvpaRa/vFBV59z8wezeY7EEfk -vW+KBLKl5qQM4xm8B7K2dz4o60L3lXNJyE6UaCavaRNis3a0q5rIFlTiNCKDTniUHdiHED -EYyC4uAC/2aSgXZdW/+uythxLVsRmpA3weFrMAfMRCM45Luxj2N8LZ4R213aPikX+dSMHK -hxnxvGL9zDwMRR3v4jqBYqXaLPpZKedyvrcNc3TqG+RhUsuSU/0j5YEk683zTxWmIc4Ze6 -j/PDXFrSRREwZCuzeL4emkDUSUKdh0FJ1vCEDhT+DuptAAAAgAdM+SFIjOjMd4L+7GidVF -VE3f+OL5Z6rPRNmwx4ifibukHdk9wsHp55yfccxr013xvM/EOmnhNqb5uNj/MgN981/ZFb -RJobRG4NuH1HHZMd9F8BE5JtkUiuuhryxEzfwTpsgEJuCtR5BW8v9laKYkaBViejZ/XM2M -bjIKCeOooOAAAAgQDarA/9tVL2YR0VupZfQQdZL+4lO2RtvddjdSKEECvckxSRIKVA/9HY -KJRHwrNIzA8sPuxNBR+9BRp+cw3cgTIQ/Rp+ISF9PFVxTMnM//vcsUTUxziYvaal3DDIpb -f8KcPJ6ECSVq/TYPYQ+wCLoPM49c8TnW8PxtDb3f+fI5WBdwAAAIEA2lT6BkcR9A6FHB+j -Ad2PpLYJqef1ZCQJrhvQRlQfrqhmAie7TdTWi7eH/jUksh+4j4rlobQ22ryTBKMA91npq8 -8lhuJnULmWzl7CBTR8YQ/6WbY58PTadkk79e9w/4WOWp8b0SxMwtZDnESs5x1dfuUyqqfS -rFBHiFOY1MK2F7sAAAAdbXNvaGFuaUBNb3MtTWFjQm9vay1Qcm8ubG9jYWwBAgMEBQY= ------END OPENSSH PRIVATE KEY-----""" + diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index b9f253b9aa..1a88554e39 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +import hashlib import os from celery import shared_task @@ -7,7 +8,8 @@ import datetime import paramiko import logging -from tdpservice.data_files.models import DataFile +from tdpservice.data_files.models import DataFile, LegacyFileTransfer + logger = logging.getLogger(__name__) server_address = settings.SERVER_ADDRESS @@ -33,7 +35,14 @@ def upload(data_file_pk): to sftp server as defined in Settings file """ data_file = DataFile.objects.get(id=data_file_pk) - destination = str(data_file.file).split('/')[-1] + file_transfer_record = LegacyFileTransfer( + data_file=data_file, + uploaded_by=data_file.user, + file_name=data_file.create_filename(), + ) + logger.info(file_transfer_record) + destination = str(data_file.create_filename()) + logger.info(destination) today_date = datetime.datetime.today() upper_directory_name = today_date.strftime('%Y%m%d') lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) @@ -49,17 +58,22 @@ def upload(data_file_pk): f = data_file.file.read() with open(destination, 'wb') as f1: f1.write(f) + file_transfer_record.file_size = f1.tell() + file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() f1.close() logger.info(os.listdir()) # Upload file try: - # sftp = transport.open_sftp() # temp file can be the file object sftp.put(destination, upper_directory_name + '/' + lower_directory_name + '/' + destination) os.remove(destination) logger.info('File {} has been successfully uploaded to {}'.format(destination, server_address)) + file_transfer_record.result = LegacyFileTransfer.Result.COMPLETED + file_transfer_record.save() return True except Exception as e: logger.error('Failed to upload {} with error:{}'.format(destination, e)) + file_transfer_record.result = LegacyFileTransfer.Result.ERROR + file_transfer_record.save() return False diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index f93205c5a7..6ecaa8c7bb 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -3,7 +3,6 @@ from celery import Celery, shared_task from django.conf import settings import configurations -from celery.schedules import crontab # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.local") @@ -20,17 +19,3 @@ # Load task modules from all registered Django apps. app.autodiscover_tasks() - - -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) - - -app.conf.beat_schedule = { - 'example': { - 'task': 'tdpservice.scheduling.tasks.run_backup', - 'schedule': crontab(minute='10'), - 'args': ['lolo'], - }, -} From 7a92e00d1546635734c00f247e1f3e24875d6480 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 29 Jul 2022 08:24:50 -0400 Subject: [PATCH 022/113] 1133 - Removed the tasks app --- tdrs-backend/tasks/__init__.py | 0 tdrs-backend/tasks/migrations/__init__.py | 0 tdrs-backend/tasks/models.py | 3 --- tdrs-backend/tasks/tests.py | 3 --- tdrs-backend/tasks/views.py | 3 --- 5 files changed, 9 deletions(-) delete mode 100644 tdrs-backend/tasks/__init__.py delete mode 100644 tdrs-backend/tasks/migrations/__init__.py delete mode 100644 tdrs-backend/tasks/models.py delete mode 100644 tdrs-backend/tasks/tests.py delete mode 100644 tdrs-backend/tasks/views.py diff --git a/tdrs-backend/tasks/__init__.py b/tdrs-backend/tasks/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tdrs-backend/tasks/migrations/__init__.py b/tdrs-backend/tasks/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tdrs-backend/tasks/models.py b/tdrs-backend/tasks/models.py deleted file mode 100644 index 71a8362390..0000000000 --- a/tdrs-backend/tasks/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/tdrs-backend/tasks/tests.py b/tdrs-backend/tasks/tests.py deleted file mode 100644 index 7ce503c2dd..0000000000 --- a/tdrs-backend/tasks/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/tdrs-backend/tasks/views.py b/tdrs-backend/tasks/views.py deleted file mode 100644 index 91ea44a218..0000000000 --- a/tdrs-backend/tasks/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. From 7f190c239a50a472b796653421fc636f383575cc Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 29 Jul 2022 08:39:58 -0400 Subject: [PATCH 023/113] 1133 - Linting --- tdrs-backend/tdpservice/data_files/views.py | 4 ++-- tdrs-backend/tdpservice/scheduling/admin.py | 2 +- tdrs-backend/tdpservice/scheduling/apps.py | 2 ++ tdrs-backend/tdpservice/scheduling/tasks.py | 6 +++--- .../scheduling/test/test_file_upload.py | 17 +---------------- tdrs-backend/tdpservice/settings/celery.py | 2 +- 6 files changed, 10 insertions(+), 23 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 8b3282a0f2..1852b98bf6 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -17,6 +17,7 @@ from tdpservice.data_files.serializers import DataFileSerializer from tdpservice.data_files.models import DataFile from tdpservice.users.permissions import DataFilePermissions +from tdpservice.scheduling import tasks logger = logging.getLogger() @@ -34,8 +35,6 @@ class Meta: fields = ['stt', 'quarter', 'year'] -from tdpservice.scheduling import tasks - class DataFileViewSet(ModelViewSet): """Data file views.""" @@ -55,6 +54,7 @@ class DataFileViewSet(ModelViewSet): ordering = ['-version'] def create(self, request, *args, **kwargs): + """Override create to upload in case of successful scan.""" response = super().create(request, *args, **kwargs) # Upload to ACF-TITAN only if file is passed the virus scan and created diff --git a/tdrs-backend/tdpservice/scheduling/admin.py b/tdrs-backend/tdpservice/scheduling/admin.py index 8c38f3f3da..6f216cfbc4 100644 --- a/tdrs-backend/tdpservice/scheduling/admin.py +++ b/tdrs-backend/tdpservice/scheduling/admin.py @@ -1,3 +1,3 @@ -from django.contrib import admin +"""Admin class for Scheduling tasks objects.""" # Register your models here. diff --git a/tdrs-backend/tdpservice/scheduling/apps.py b/tdrs-backend/tdpservice/scheduling/apps.py index 8e7fd7a29a..f0c9e5d658 100644 --- a/tdrs-backend/tdpservice/scheduling/apps.py +++ b/tdrs-backend/tdpservice/scheduling/apps.py @@ -1,3 +1,5 @@ +"""Scheduling app configuration.""" + from django.apps import AppConfig diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 1a88554e39..52af6b7185 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -19,7 +19,7 @@ def write_key_to_file(private_key): """ - Paramiko require the key in file object format + Paramiko require the key in file object format. """ with open('temp_key_file', 'w') as f: f.write(private_key) @@ -30,8 +30,8 @@ def write_key_to_file(private_key): @shared_task def upload(data_file_pk): """ - This task uploads the file in DataFile object with - pk = data_file_pk + Upload to SFTP server. + This task uploads the file in DataFile object with pk = data_file_pk to sftp server as defined in Settings file """ data_file = DataFile.objects.get(id=data_file_pk) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index e365d9cea3..bc1ab17ba2 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -1,21 +1,6 @@ """Integration test(s) for clamav-rest operations.""" -from os import remove -from requests.sessions import Session -import pytest - -from django.contrib.admin import site as admin_site -from django.core.files.base import File -from rest_framework.status import HTTP_400_BAD_REQUEST -from tdpservice.scheduling.tasks import upload - -""" -WRITE TEST: - -1. CREATE FILE -> clean_file fixture -> DONE -2. UPLOAD TO ACF_TITAN -> -3. SSH TO ACF TITAN AND MAKE SURE THE FILE EXISTS -""" +import pytest from tdpservice.data_files.test.factories import DataFileFactory diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 6ecaa8c7bb..212cd1d8c0 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,7 +1,7 @@ +"""Celey configuration file.""" from __future__ import absolute_import import os from celery import Celery, shared_task -from django.conf import settings import configurations # Set the default Django settings module for the 'celery' program. From 33276b0b6ea1634ed321ec493c1efa871a60f55b Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 29 Jul 2022 09:10:53 -0400 Subject: [PATCH 024/113] 1133 - Linting again --- tdrs-backend/tdpservice/scheduling/tasks.py | 5 ++--- tdrs-backend/tdpservice/scheduling/test/test_file_upload.py | 2 ++ tdrs-backend/tdpservice/settings/celery.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 52af6b7185..e385c9a0a1 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -18,9 +18,7 @@ def write_key_to_file(private_key): - """ - Paramiko require the key in file object format. - """ + """Paramiko require the key in file object format.""" with open('temp_key_file', 'w') as f: f.write(private_key) f.close() @@ -31,6 +29,7 @@ def write_key_to_file(private_key): def upload(data_file_pk): """ Upload to SFTP server. + This task uploads the file in DataFile object with pk = data_file_pk to sftp server as defined in Settings file """ diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index bc1ab17ba2..7fc6368e28 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -7,9 +7,11 @@ @pytest.fixture() def upload_file(): + """Fixture for test upload file.""" return DataFileFactory() @pytest.mark.django_db def test_upload_server(upload_file): + """Test upload file.""" assert upload_file.original_filename == 'data_file.txt' diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 212cd1d8c0..d66f30f89c 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,7 +1,7 @@ """Celey configuration file.""" from __future__ import absolute_import import os -from celery import Celery, shared_task +from celery import Celery import configurations # Set the default Django settings module for the 'celery' program. From 2f0bc798615e1cf4e6335ed45f919e3f32d73fdd Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 29 Jul 2022 09:19:26 -0400 Subject: [PATCH 025/113] 1133 - Linting --- tdrs-backend/tdpservice/scheduling/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/scheduling/apps.py b/tdrs-backend/tdpservice/scheduling/apps.py index f0c9e5d658..94cc68eee8 100644 --- a/tdrs-backend/tdpservice/scheduling/apps.py +++ b/tdrs-backend/tdpservice/scheduling/apps.py @@ -4,5 +4,6 @@ class TasksConfig(AppConfig): + """Scheduling task config.""" default_auto_field = 'django.db.models.BigAutoField' name = 'tdpservice.scheduling' From 4bba52020389b1f5f6e7e84faa32b8a9f904c786 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 29 Jul 2022 09:27:40 -0400 Subject: [PATCH 026/113] 1133 - Linting --- tdrs-backend/tdpservice/scheduling/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/scheduling/apps.py b/tdrs-backend/tdpservice/scheduling/apps.py index 94cc68eee8..4859cc81c8 100644 --- a/tdrs-backend/tdpservice/scheduling/apps.py +++ b/tdrs-backend/tdpservice/scheduling/apps.py @@ -5,5 +5,6 @@ class TasksConfig(AppConfig): """Scheduling task config.""" + default_auto_field = 'django.db.models.BigAutoField' name = 'tdpservice.scheduling' From eaa8e3284d2aad3a57ce9df331ed6019c8378d1f Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 29 Jul 2022 09:46:20 -0400 Subject: [PATCH 027/113] 1133 - Linting --- tdrs-backend/tdpservice/scheduling/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index e385c9a0a1..8e9fdda078 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,3 +1,4 @@ +"""Celery task configurations.""" from __future__ import absolute_import import hashlib From 25adae8bc19ef200702ca17ecb15c104689cf014 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 10:12:58 -0400 Subject: [PATCH 028/113] added redis-cli apt --- tdrs-backend/apt.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/apt.yml b/tdrs-backend/apt.yml index af1b07caa1..8ce7c2abe1 100644 --- a/tdrs-backend/apt.yml +++ b/tdrs-backend/apt.yml @@ -5,3 +5,4 @@ repos: - deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main packages: - postgresql-client-12 + - redis-tools From 319b46d89338ee2706846aa67040935334055898 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 10:57:10 -0400 Subject: [PATCH 029/113] added dependency packages --- tdrs-backend/apt.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/apt.yml b/tdrs-backend/apt.yml index 8ce7c2abe1..a068cf583d 100644 --- a/tdrs-backend/apt.yml +++ b/tdrs-backend/apt.yml @@ -5,4 +5,5 @@ repos: - deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main packages: - postgresql-client-12 + - libjemalloc-dev - redis-tools From 91384b7e47f71a4400a3f6b4362f6527fbc5321d Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 11:33:41 -0400 Subject: [PATCH 030/113] added local redis --- tdrs-backend/apt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/apt.yml b/tdrs-backend/apt.yml index a068cf583d..07229ce487 100644 --- a/tdrs-backend/apt.yml +++ b/tdrs-backend/apt.yml @@ -6,4 +6,4 @@ repos: packages: - postgresql-client-12 - libjemalloc-dev - - redis-tools + - redis From 17cad9c7fc320099f754cf9a16e967c4db575d4d Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 12:27:24 -0400 Subject: [PATCH 031/113] 1133 - Added redis-server to gunicorn_start.sh and changed the Celery broker/backend to run on the local redis-server --- tdrs-backend/gunicorn_start.sh | 4 ++++ tdrs-backend/tdpservice/settings/common.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 9b7f67de34..5de9475a26 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -7,6 +7,10 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput +# Run redis server +export LD_LIBRARY_PATH=~/deps/0/lib/ +./deps/0/bin/redis-server & + # celery -A tdpservice.settings worker -l info & sleep 5 diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 806efea005..9ea88ae229 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -405,8 +405,8 @@ class Common(Configuration): USERNAME = os.getenv('ACFTITAN_USERNAME', '') # CELERY CONFIG - CELERY_BROKER_URL = "redis://redis-server:6379" - CELERY_RESULT_BACKEND = "redis://redis-server:6379" + CELERY_BROKER_URL = "redis://localhost:6379" + CELERY_RESULT_BACKEND = "redis://localhost:6379" CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' From 59566502c3d39bc169cb9dc1a9a12b3d35e1f541 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 12:47:38 -0400 Subject: [PATCH 032/113] 1133 - Removed the test files --- .../scheduling/test/test_file_upload.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index 7fc6368e28..fb9bf7fe42 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -1,17 +1 @@ """Integration test(s) for clamav-rest operations.""" - -import pytest - -from tdpservice.data_files.test.factories import DataFileFactory - - -@pytest.fixture() -def upload_file(): - """Fixture for test upload file.""" - return DataFileFactory() - - -@pytest.mark.django_db -def test_upload_server(upload_file): - """Test upload file.""" - assert upload_file.original_filename == 'data_file.txt' From ac202a5ceca1c84c3deddfda6f1801e9309e321a Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 13:04:33 -0400 Subject: [PATCH 033/113] 1133 - Refactored redis URI to env vars --- tdrs-backend/tdpservice/settings/common.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 9ea88ae229..38790087ad 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -405,8 +405,13 @@ class Common(Configuration): USERNAME = os.getenv('ACFTITAN_USERNAME', '') # CELERY CONFIG - CELERY_BROKER_URL = "redis://localhost:6379" - CELERY_RESULT_BACKEND = "redis://localhost:6379" + REDIS_URI = os.getenv( + 'REDIS_URI', + 'redis://redis-server:6379' + ) + + CELERY_BROKER_URL = REDIS_URI + CELERY_RESULT_BACKEND = REDIS_URI CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' From e4fb775092d5d4e73ba88ab9affbc545e6d0cea5 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 22:10:16 -0400 Subject: [PATCH 034/113] 1133: absolute path instead of home path --- tdrs-backend/gunicorn_start.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 5de9475a26..273d796655 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -8,13 +8,13 @@ python manage.py populate_stts python manage.py collectstatic --noinput # Run redis server -export LD_LIBRARY_PATH=~/deps/0/lib/ -./deps/0/bin/redis-server & +export LD_LIBRARY_PATH=~/deps/0/lib/:LD_LIBRARY_PATH +./home/vcap/deps/0/bin/redis-server & # celery -A tdpservice.settings worker -l info & sleep 5 -celery -A tdpservice.settings --broker=redis://redis-server:6379 flower & +celery -A tdpservice.settings --broker=redis://$REDIS_URI:6379 flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & echo "Starting Gunicorn" From 56559e68caa08895d7018011dbf2003f6d29ef4c Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 22:36:38 -0400 Subject: [PATCH 035/113] 1133: absolute path instead of home path --- tdrs-backend/gunicorn_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 273d796655..d4ab8ce0dc 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -8,7 +8,7 @@ python manage.py populate_stts python manage.py collectstatic --noinput # Run redis server -export LD_LIBRARY_PATH=~/deps/0/lib/:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:LD_LIBRARY_PATH ./home/vcap/deps/0/bin/redis-server & # From d31af210bd5a05bcc2110986690f5263c251dbcf Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 22:38:44 -0400 Subject: [PATCH 036/113] 1133: corrected flower uri --- tdrs-backend/gunicorn_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index d4ab8ce0dc..79ae6cb35a 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -14,7 +14,7 @@ export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:LD_LIBRARY_PATH # celery -A tdpservice.settings worker -l info & sleep 5 -celery -A tdpservice.settings --broker=redis://$REDIS_URI:6379 flower & +celery -A tdpservice.settings --broker=redis://localhost:6379 flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & echo "Starting Gunicorn" From b518124b3bc03a6784210b213e0810e7f73908ea Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 22:56:53 -0400 Subject: [PATCH 037/113] 1133: corrected gunicorn_start.sh --- tdrs-backend/gunicorn_start.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 79ae6cb35a..6563127d0c 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -1,17 +1,18 @@ #!/usr/bin/env bash # Apply database migrations set -e + +echo "Run redis server" +export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:LD_LIBRARY_PATH +./home/vcap/deps/0/bin/redis-server & + +# echo "Applying database migrations" python manage.py makemigrations python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput -# Run redis server -export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:LD_LIBRARY_PATH -./home/vcap/deps/0/bin/redis-server & - -# celery -A tdpservice.settings worker -l info & sleep 5 celery -A tdpservice.settings --broker=redis://localhost:6379 flower & From dd2d8f268f654d9dda73af7ae9444ff0787961cd Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 23:31:20 -0400 Subject: [PATCH 038/113] 1133: corrected gunicorn_start.sh --- tdrs-backend/gunicorn_start.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 6563127d0c..c6879d401e 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -3,8 +3,8 @@ set -e echo "Run redis server" -export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:LD_LIBRARY_PATH -./home/vcap/deps/0/bin/redis-server & +export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/ +( cd /home/vcap/deps/0/bin/ ; ./redis-server &) # echo "Applying database migrations" From d1b31305a95f5cf8f4ac779754fff22d50edb02e Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 23:48:58 -0400 Subject: [PATCH 039/113] 1133: corrected gunicorn_start.sh --- tdrs-backend/gunicorn_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index c6879d401e..2fa820422c 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -3,7 +3,7 @@ set -e echo "Run redis server" -export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/ +export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:$LD_LIBRARY_PATH ( cd /home/vcap/deps/0/bin/ ; ./redis-server &) # From 51c91de4503375e71c4b1afa9fb563830bfb323d Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 1 Aug 2022 23:52:07 -0400 Subject: [PATCH 040/113] 1133: corrected gunicorn_start.sh --- tdrs-backend/gunicorn_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 2fa820422c..30cd447d29 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -3,7 +3,7 @@ set -e echo "Run redis server" -export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:/home/vcap/deps/1/lib:$LD_LIBRARY_PATH ( cd /home/vcap/deps/0/bin/ ; ./redis-server &) # From 4a83bba95a078082680b7a5c2fb8953311c0a4cb Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 2 Aug 2022 10:24:26 -0400 Subject: [PATCH 041/113] 1133: wrapped the task in try/exception --- tdrs-backend/tdpservice/scheduling/tasks.py | 54 ++++++++++----------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 8e9fdda078..3a3ebf0b79 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -34,37 +34,37 @@ def upload(data_file_pk): This task uploads the file in DataFile object with pk = data_file_pk to sftp server as defined in Settings file """ - data_file = DataFile.objects.get(id=data_file_pk) - file_transfer_record = LegacyFileTransfer( - data_file=data_file, - uploaded_by=data_file.user, - file_name=data_file.create_filename(), - ) - logger.info(file_transfer_record) - destination = str(data_file.create_filename()) - logger.info(destination) - today_date = datetime.datetime.today() - upper_directory_name = today_date.strftime('%Y%m%d') - lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) + # Upload file + try: + data_file = DataFile.objects.get(id=data_file_pk) + file_transfer_record = LegacyFileTransfer( + data_file=data_file, + uploaded_by=data_file.user, + file_name=data_file.create_filename(), + ) + logger.info(file_transfer_record) + destination = str(data_file.create_filename()) + logger.info(destination) + today_date = datetime.datetime.today() + upper_directory_name = today_date.strftime('%Y%m%d') + lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) - transport = paramiko.SSHClient() - transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - transport.connect(server_address, key_filename=write_key_to_file(local_key), username=username) - sftp = transport.open_sftp() + transport = paramiko.SSHClient() + transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + transport.connect(server_address, key_filename=write_key_to_file(local_key), username=username) + sftp = transport.open_sftp() - transport.exec_command('mkdir -p ' + upper_directory_name + - '/' + lower_directory_name) + transport.exec_command('mkdir -p ' + upper_directory_name + + '/' + lower_directory_name) - f = data_file.file.read() - with open(destination, 'wb') as f1: - f1.write(f) - file_transfer_record.file_size = f1.tell() - file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() - f1.close() - logger.info(os.listdir()) + f = data_file.file.read() + with open(destination, 'wb') as f1: + f1.write(f) + file_transfer_record.file_size = f1.tell() + file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() + f1.close() + logger.info(os.listdir()) - # Upload file - try: # temp file can be the file object sftp.put(destination, upper_directory_name + '/' + lower_directory_name + '/' + destination) os.remove(destination) From 30d5f7f89708c470830ed054d8b3f36385755232 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 2 Aug 2022 11:27:22 -0400 Subject: [PATCH 042/113] 1133: additional log --- tdrs-backend/tdpservice/data_files/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 1852b98bf6..f74b012b4d 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -57,6 +57,7 @@ def create(self, request, *args, **kwargs): """Override create to upload in case of successful scan.""" response = super().create(request, *args, **kwargs) + logger.info('_________________________________:',response.status_code) # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED: tasks.upload.delay( From 5597c5bbee1c61bcefa2442bb537af1f77071f88 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 2 Aug 2022 11:33:32 -0400 Subject: [PATCH 043/113] 1133: additional log --- tdrs-backend/tdpservice/data_files/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index f74b012b4d..ddca63ca92 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -57,9 +57,10 @@ def create(self, request, *args, **kwargs): """Override create to upload in case of successful scan.""" response = super().create(request, *args, **kwargs) - logger.info('_________________________________:',response.status_code) + logger.info('____________________________:', response.status_code) + logger.info('____________________________:', response.data.get('id')) # Upload to ACF-TITAN only if file is passed the virus scan and created - if response.status_code == status.HTTP_201_CREATED: + if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: tasks.upload.delay( data_file_pk=response.data.get('id') ) From c3b2e65ca66ec7b9ae6a5b63f676cf85d8f33082 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 2 Aug 2022 11:54:14 -0400 Subject: [PATCH 044/113] 1133: additional log --- tdrs-backend/tdpservice/data_files/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index ddca63ca92..8889ee4012 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -58,7 +58,7 @@ def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) logger.info('____________________________:', response.status_code) - logger.info('____________________________:', response.data.get('id')) + # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: tasks.upload.delay( From 5ca7b04f8159f78c1af14cde2a15bbdefc76c6b4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 2 Aug 2022 12:03:10 -0400 Subject: [PATCH 045/113] 1133: additional log --- tdrs-backend/tdpservice/data_files/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 8889ee4012..f7d36066f1 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -57,8 +57,6 @@ def create(self, request, *args, **kwargs): """Override create to upload in case of successful scan.""" response = super().create(request, *args, **kwargs) - logger.info('____________________________:', response.status_code) - # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: tasks.upload.delay( From 3cb3055e38cd198c298fe74796d1011485dc0f74 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 4 Aug 2022 10:13:30 -0400 Subject: [PATCH 046/113] 1133: Added error handling, starting redis on cloud.gov script --- tdrs-backend/docker-compose.yml | 2 ++ tdrs-backend/gunicorn_start.sh | 14 ++++++++---- tdrs-backend/tdpservice/data_files/views.py | 2 +- tdrs-backend/tdpservice/scheduling/tasks.py | 24 ++++++++++----------- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 9d9a448a23..43a445d3db 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -70,6 +70,8 @@ services: - ACFTITAN_HOST - ACFTITAN_KEY - ACFTITAN_USERNAME + - REDIS_URI + - REDIS_SERVER_LOCAL volumes: - .:/tdpapp image: tdp diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 30cd447d29..3458834767 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -2,9 +2,15 @@ # Apply database migrations set -e -echo "Run redis server" -export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:/home/vcap/deps/1/lib:$LD_LIBRARY_PATH -( cd /home/vcap/deps/0/bin/ ; ./redis-server &) +echo "REDIS_SERVER" +echo $REDIS_SERVER_LOCAL +if [[ "$REDIS_SERVER_LOCAL" = "TRUE" ]]; then + echo "Run redis server on docker" +else + echo "Run redis server locally" + export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:/home/vcap/deps/1/lib:$LD_LIBRARY_PATH + ( cd /home/vcap/deps/0/bin/; ./redis-server &) +fi # echo "Applying database migrations" @@ -15,7 +21,7 @@ python manage.py collectstatic --noinput celery -A tdpservice.settings worker -l info & sleep 5 -celery -A tdpservice.settings --broker=redis://localhost:6379 flower & +celery -A tdpservice.settings --broker=$REDIS_URI flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & echo "Starting Gunicorn" diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index f7d36066f1..c45b881e4b 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -15,7 +15,7 @@ from rest_framework import status from tdpservice.data_files.serializers import DataFileSerializer -from tdpservice.data_files.models import DataFile +from tdpservice.data_files.models import DataFile, LegacyFileTransfer from tdpservice.users.permissions import DataFilePermissions from tdpservice.scheduling import tasks diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 3a3ebf0b79..e4b5ed9979 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -35,35 +35,34 @@ def upload(data_file_pk): to sftp server as defined in Settings file """ # Upload file + data_file = DataFile.objects.get(id=data_file_pk) + file_transfer_record = LegacyFileTransfer( + data_file=data_file, + uploaded_by=data_file.user, + file_name=data_file.create_filename(), + ) + try: - data_file = DataFile.objects.get(id=data_file_pk) - file_transfer_record = LegacyFileTransfer( - data_file=data_file, - uploaded_by=data_file.user, - file_name=data_file.create_filename(), - ) - logger.info(file_transfer_record) + logger.info(str(file_transfer_record.__dict__)) destination = str(data_file.create_filename()) logger.info(destination) today_date = datetime.datetime.today() upper_directory_name = today_date.strftime('%Y%m%d') lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) - transport = paramiko.SSHClient() transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - transport.connect(server_address, key_filename=write_key_to_file(local_key), username=username) + temp_key_file = write_key_to_file(local_key) + transport.connect(server_address, key_filename=temp_key_file, username=username) + os.remove(temp_key_file) sftp = transport.open_sftp() - transport.exec_command('mkdir -p ' + upper_directory_name + '/' + lower_directory_name) - f = data_file.file.read() with open(destination, 'wb') as f1: f1.write(f) file_transfer_record.file_size = f1.tell() file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() f1.close() - logger.info(os.listdir()) # temp file can be the file object sftp.put(destination, upper_directory_name + '/' + lower_directory_name + '/' + destination) @@ -74,6 +73,7 @@ def upload(data_file_pk): return True except Exception as e: logger.error('Failed to upload {} with error:{}'.format(destination, e)) + file_transfer_record.file_size = 0 file_transfer_record.result = LegacyFileTransfer.Result.ERROR file_transfer_record.save() return False From 5eca1dbc763261a50f1235cd636b84388b6ad05a Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 4 Aug 2022 10:23:41 -0400 Subject: [PATCH 047/113] 1133: Linting --- tdrs-backend/tdpservice/data_files/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index c45b881e4b..f7d36066f1 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -15,7 +15,7 @@ from rest_framework import status from tdpservice.data_files.serializers import DataFileSerializer -from tdpservice.data_files.models import DataFile, LegacyFileTransfer +from tdpservice.data_files.models import DataFile from tdpservice.users.permissions import DataFilePermissions from tdpservice.scheduling import tasks From 63ad4c12ec02122699701d57f3b2efc03a7127c1 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 15 Aug 2022 10:55:19 -0400 Subject: [PATCH 048/113] 1133: Upload Pytest, and linting --- tdrs-backend/Pipfile | 14 +- tdrs-backend/Pipfile.lock | 328 +++++++++--------- tdrs-backend/tdpservice/scheduling/tasks.py | 45 ++- .../scheduling/test/test_file_upload.py | 100 +++++- 4 files changed, 296 insertions(+), 191 deletions(-) diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index e50d2967f9..6112980b91 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -20,7 +20,8 @@ pytest-mock = "*" pytest-factoryboy = "*" [packages] -boto3 = "==1.17.112" +pytz = "2022.1" +boto3 = "==1.24.51" cryptography = "==3.4.7" dj-database-url = "==0.5.0" django = "==3.2.13" @@ -45,11 +46,12 @@ pyjwt = "==2.4.0" requests = "==2.27.1" wait-for-it = "==2.2.0" requests-mock = "==1.9.3" -celery = "*" -redis = "*" -flower = "1.1.0" -django-celery-beat = "2.3.0" -paramiko = "2.11.0" +celery = "==5.2.6" +redis = "==4.1.2" +flower = "==1.1.0" +django-celery-beat = "==2.2.1" +paramiko = "==2.11.0" +pytest_sftpserver = "==1.3.0" [requires] python_version = "3.10.4" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index c0b4efa4bb..156b6f3921 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "48ccbab3ee5d988f00d302c10a62be2e8261eb3d617dcb31c39c0222436abcc7" + "sha256": "27fb1791e4b918216b42f4bf4effc6daa3f1962709e487ef901bf5342762f9da" }, "pipfile-spec": 6, "requires": { @@ -24,14 +24,6 @@ "markers": "python_version >= '3.6'", "version": "==5.1.1" }, - "appnope": { - "hashes": [ - "sha256:02bd91c4de869fbb1e1c50aafc4098827a7a54ab2f39d9dcba6c9547ed920e24", - "sha256:265a455292d0bd8a72453494fa24df5a11eb18373a60c7c0430889f22548605e" - ], - "markers": "sys_platform == 'darwin'", - "version": "==0.1.3" - }, "asgiref": { "hashes": [ "sha256:1d2880b792ae8757289136f1db2b7b99100ce959b2aa57fd69dab783d05afac4", @@ -42,18 +34,10 @@ }, "asttokens": { "hashes": [ - "sha256:0844691e88552595a6f4a4281a9f7f79b8dd45ca4ccea82e5e05b4bbdb76705c", - "sha256:9a54c114f02c7a9480d56550932546a3f1fe71d8a02f1bc7ccd0ee3ee35cf4d5" + "sha256:8444353e4e2a99661c8dfb85ec9c02eedded08f0006234bff7db44a06840acc2", + "sha256:f5589ef8518f73dd82c15e1c19f795d8a62c133485e557c04443d4a1a730cf9f" ], - "version": "==2.0.5" - }, - "async-timeout": { - "hashes": [ - "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15", - "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c" - ], - "markers": "python_version >= '3.6'", - "version": "==4.0.2" + "version": "==2.0.7" }, "backcall": { "hashes": [ @@ -88,27 +72,27 @@ }, "boto3": { "hashes": [ - "sha256:08b6dacbe7ebe57ae8acfb7106b2728d946ae1e0c3da270caee1deb79ccbd8af", - "sha256:8716465313c50ad9e5c2ac1767642ca0ddf7d1729c3d5c884d82880c1a15a310" + "sha256:0f1689e487ea093be8a124cd5e9766cfcf2c153fc2af295c3853c5b63e2a73ed", + "sha256:ca8352b4af67cf15a8f399f71dfc31c6a0e5a7725afcf299c50aaaaa9f1e6782" ], "index": "pypi", - "version": "==1.17.112" + "version": "==1.24.51" }, "botocore": { "hashes": [ - "sha256:6d51de0981a3ef19da9e6a3c73b5ab427e3c0c8b92200ebd38d087299683dd2b", - "sha256:d0b9b70b6eb5b65bb7162da2aaf04b6b086b15cc7ea322ddc3ef2f5e07944dcf" + "sha256:328d2baca30e66016acdf9ad3c5e6fa6522fca249f54c5affce8774c0faa564f", + "sha256:4ba1678bc78fbdac9a7e1c010b057ad9ed96dc6534d1c82718af08d746b652ed" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.20.112" + "markers": "python_version >= '3.7'", + "version": "==1.27.51" }, "celery": { "hashes": [ - "sha256:138420c020cd58d6707e6257b6beda91fd39af7afde5d36c6334d175302c0e14", - "sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d" + "sha256:d1398cadf30f576266b34370e28e880306ec55f7a4b6307549b0ae9c15663481", + "sha256:da31f8eae7607b1582e5ee2d3f2d6f58450585afd23379491e3d9229d08102d0" ], "index": "pypi", - "version": "==5.2.7" + "version": "==5.2.6" }, "certifi": { "hashes": [ @@ -208,7 +192,7 @@ "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" ], - "markers": "python_full_version >= '3.6.2' and python_full_version < '4.0.0'", + "markers": "python_version < '4' and python_full_version >= '3.6.2'", "version": "==0.3.0" }, "click-plugins": { @@ -264,7 +248,7 @@ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186" ], - "markers": "python_version >= '3.5'", + "markers": "python_version >= '3.7'", "version": "==5.1.1" }, "deprecated": { @@ -309,18 +293,18 @@ }, "django-celery-beat": { "hashes": [ - "sha256:ba24f86ed956ba0ee2b43237a49303ebf5aa7e0fb0ccb7e0542b7e31a463dcba", - "sha256:ff694e1d30f84de3d2b78822c90005c6c7c1462768e845ae162af71a6a46f1eb" + "sha256:97ae5eb309541551bdb07bf60cc57cadacf42a74287560ced2d2c06298620234", + "sha256:ab43049634fd18dc037927d7c2c7d5f67f95283a20ebbda55f42f8606412e66c" ], "index": "pypi", - "version": "==2.3.0" + "version": "==2.2.1" }, "django-colorfield": { "hashes": [ - "sha256:08d093d5b31b599b681665ab97b957045c4a4c0b93cfc434bd257378a7193c96", - "sha256:a58d1c5a56f0380439dc1305a131321b839df35ddfac2a6dbcc887de02232120" + "sha256:29dffa09b2ce89f19504a53a120e5bfdbfe3e44621f33b3f94b0e6a572ae93c1", + "sha256:9c2c36a7cac60bf6e57e70ca344529a3065226543358d01036e1714371f3166d" ], - "version": "==0.7.1" + "version": "==0.7.2" }, "django-configurations": { "hashes": [ @@ -392,11 +376,11 @@ }, "django-timezone-field": { "hashes": [ - "sha256:15746ed367a5a32eda76cfa2886eeec1de8cda79f519b7c5e12f87ed7cdbd663", - "sha256:199f211082eeac7e83563929b8ce41399c1c0f00dfc2f36bc00bea381027eaaa" + "sha256:5dd5bd9249382bef8847d3e7e4c32b7be182a4b538f354130d1252ed228892f8", + "sha256:7552d2b0f145684b7de3fb5046101c7efd600cc6ba951b15c630fa1e1b83558e" ], - "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", - "version": "==5.0" + "markers": "python_version >= '3.5'", + "version": "==4.2.3" }, "django-unique-upload": { "hashes": [ @@ -424,10 +408,10 @@ }, "executing": { "hashes": [ - "sha256:c6554e21c6b060590a6d3be4b82fb78f8f0194d809de5ea7df1c093763311501", - "sha256:d1eef132db1b83649a3905ca6dd8897f71ac6f8cac79a7e58a1a09cf137546c9" + "sha256:9c745f80cda11eb22b62cbecf21156491a794eb56ab06f9d286a44e62822b24e", + "sha256:d1cd87c2e371e9966261410c5b3769d6df2f9e4a79a83eebd2662dd3388f9833" ], - "version": "==0.8.3" + "version": "==0.10.0" }, "flower": { "hashes": [ @@ -447,11 +431,11 @@ }, "humanize": { "hashes": [ - "sha256:2bc1fdd831cd00557d3010abdd84d3e41b4a96703a3eaf6c24ee290b26b75a44", - "sha256:bed628920d45cd5018abb095710f0c03a8336d6ac0790e7647c6a328f3880b81" + "sha256:0dfac79fe8c1c0c734c14177b07b857bad9ae30dd50daa0a14e2c3d8054ee0c4", + "sha256:5dd159c9910cd57b94072e4d7decae097f0eb84c4645153706929a7f127cb2ef" ], "markers": "python_version >= '3.7'", - "version": "==4.2.3" + "version": "==4.3.0" }, "idna": { "hashes": [ @@ -481,7 +465,7 @@ "sha256:7ca74052a38fa25fe9bedf52da0be7d3fdd2fb027c3b778ea78dfe8c212937d1", "sha256:f2db3a10254241d9b447232cec8b424847f338d9d36f9a577a6192c332a46abd" ], - "markers": "python_version >= '3.8'", + "markers": "python_version >= '3.7'", "version": "==8.4.0" }, "itypes": { @@ -509,11 +493,11 @@ }, "jmespath": { "hashes": [ - "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", - "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.10.0" + "markers": "python_version >= '3.7'", + "version": "==1.0.1" }, "jwcrypto": { "hashes": [ @@ -834,6 +818,14 @@ "markers": "python_full_version >= '3.6.8'", "version": "==3.0.9" }, + "pytest-sftpserver": { + "hashes": [ + "sha256:b7ac34a23f63d77e27f67b6a81c9418243733f027eeb8a3061d965b2da7e5cab", + "sha256:c5e8a37049866d4eabc711db9f1c09e1c02ab72ba290f5fd244939c9a188042f" + ], + "index": "pypi", + "version": "==1.3.0" + }, "python-crontab": { "hashes": [ "sha256:1e35ed7a3cdc3100545b43e196d34754e6551e7f95e4caebbe0e1c0ca41c2f1b" @@ -850,18 +842,19 @@ }, "pytz": { "hashes": [ - "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", - "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197", + "sha256:cea221417204f2d1a2aa03ddae3e867921971d0d76f14d87abb4414415bbdcf5" ], - "version": "==2022.1" + "index": "pypi", + "version": "==2022.2.1" }, "redis": { "hashes": [ - "sha256:a52d5694c9eb4292770084fa8c863f79367ca19884b329ab574d5cb2036b3e54", - "sha256:ddf27071df4adf3821c4f2ca59d67525c3a82e5f268bed97b813cb4fabf87880" + "sha256:bf86397be532fc0a888d5976a5313a3a70d8f912d52bc0c09bffda4b8425a1d4", + "sha256:f13eea4254e302485add677cadedaf1305c1b3a4e07535e23b7b239798ce9301" ], "index": "pypi", - "version": "==4.3.4" + "version": "==4.1.2" }, "requests": { "hashes": [ @@ -920,18 +913,19 @@ }, "s3transfer": { "hashes": [ - "sha256:9b3752887a2880690ce628bc263d6d13a3864083aeacff4890c1c9839a5eb0bc", - "sha256:cb022f4b16551edebbb31a377d3f09600dbada7363d8c5db7976e7f47732e1b2" + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" ], - "version": "==0.4.2" + "markers": "python_version >= '3.7'", + "version": "==0.6.0" }, "setuptools": { "hashes": [ - "sha256:0d33c374d41c7863419fc8f6c10bfe25b7b498aa34164d135c622e52580c6b16", - "sha256:c04b44a57a6265fe34a4a444e965884716d34bae963119a76353434d6f18e450" + "sha256:d73f8cd714a1a6691f5eb5abeeacbf313242b7aa2f5eba93776542c1aad90c6f", + "sha256:fe9a97f68b064a6ddd4bacfb0b4b93a4c65a556d97ce906255540439d0c35cef" ], "markers": "python_version >= '3.7'", - "version": "==63.2.0" + "version": "==65.0.0" }, "six": { "hashes": [ @@ -951,17 +945,17 @@ }, "stack-data": { "hashes": [ - "sha256:77bec1402dcd0987e9022326473fdbcc767304892a533ed8c29888dacb7dddbc", - "sha256:aa1d52d14d09c7a9a12bb740e6bdfffe0f5e8f4f9218d85e7c73a8c37f7ae38d" + "sha256:a90ae7e260f7d15aefeceb46f0a028d4ccb9eb8856475c53e341945342d41ea7", + "sha256:b94fed36d725cfabc6d09ed5886913e35eed9009766a1af1d5941b9da3a94aaa" ], - "version": "==0.3.0" + "version": "==0.4.0" }, "toml": { "hashes": [ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '3.7'", "version": "==0.10.2" }, "tornado": { @@ -989,14 +983,6 @@ "markers": "python_version >= '3.7'", "version": "==5.3.0" }, - "tzdata": { - "hashes": [ - "sha256:238e70234214138ed7b4e8a0fab0e5e13872edab3be586ab8198c407620e2ab9", - "sha256:8b536a8ec63dc0751342b3984193a3118f8fca2afe25752bb9b7fffd398552d3" - ], - "markers": "python_version >= '2'", - "version": "==2022.1" - }, "uritemplate": { "hashes": [ "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", @@ -1007,11 +993,11 @@ }, "urllib3": { "hashes": [ - "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", - "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", - "version": "==1.26.10" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.11" }, "vine": { "hashes": [ @@ -1110,19 +1096,19 @@ "develop": { "attrs": { "hashes": [ - "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", - "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", + "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.4.0" + "markers": "python_version >= '3.5'", + "version": "==22.1.0" }, "awscli": { "hashes": [ - "sha256:671e735d7efb052092f38dc453398c37cbfe191ee4ecbf18cfd6d3f98e8ad629", - "sha256:9a049a0bde495954c14676ec2750a1acaf00240e4bc53282acc6db0a40ee748b" + "sha256:6d11111504bbdc4f8abd7b2f77c7682333a07b172abfb5f46123119594bfa25d", + "sha256:f1f1fd96b8a7dcbb6cb79d5d39c7eb524b2dc73c40441f31ae65238308f80a0f" ], "index": "pypi", - "version": "==1.25.30" + "version": "==1.25.51" }, "awscli-local": { "hashes": [ @@ -1133,19 +1119,19 @@ }, "boto3": { "hashes": [ - "sha256:08b6dacbe7ebe57ae8acfb7106b2728d946ae1e0c3da270caee1deb79ccbd8af", - "sha256:8716465313c50ad9e5c2ac1767642ca0ddf7d1729c3d5c884d82880c1a15a310" + "sha256:0f1689e487ea093be8a124cd5e9766cfcf2c153fc2af295c3853c5b63e2a73ed", + "sha256:ca8352b4af67cf15a8f399f71dfc31c6a0e5a7725afcf299c50aaaaa9f1e6782" ], "index": "pypi", - "version": "==1.17.112" + "version": "==1.24.51" }, "botocore": { "hashes": [ - "sha256:6d51de0981a3ef19da9e6a3c73b5ab427e3c0c8b92200ebd38d087299683dd2b", - "sha256:d0b9b70b6eb5b65bb7162da2aaf04b6b086b15cc7ea322ddc3ef2f5e07944dcf" + "sha256:328d2baca30e66016acdf9ad3c5e6fa6522fca249f54c5affce8774c0faa564f", + "sha256:4ba1678bc78fbdac9a7e1c010b057ad9ed96dc6534d1c82718af08d746b652ed" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.20.112" + "markers": "python_version >= '3.7'", + "version": "==1.27.51" }, "click": { "hashes": [ @@ -1168,50 +1154,50 @@ "toml" ], "hashes": [ - "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32", - "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7", - "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996", - "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55", - "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46", - "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de", - "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039", - "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee", - "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1", - "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f", - "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63", - "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083", - "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe", - "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0", - "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6", - "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe", - "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933", - "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0", - "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c", - "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07", - "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8", - "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b", - "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e", - "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120", - "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f", - "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e", - "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd", - "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f", - "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386", - "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8", - "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae", - "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc", - "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783", - "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d", - "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c", - "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97", - "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978", - "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf", - "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29", - "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39", - "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452" + "sha256:04010af3c06ce2bfeb3b1e4e05d136f88d88c25f76cd4faff5d1fd84d11581ea", + "sha256:05de0762c1caed4a162b3e305f36cf20a548ff4da0be6766ad5c870704be3660", + "sha256:068d6f2a893af838291b8809c876973d885543411ea460f3e6886ac0ee941732", + "sha256:0a84376e4fd13cebce2c0ef8c2f037929c8307fb94af1e5dbe50272a1c651b5d", + "sha256:0e34247274bde982bbc613894d33f9e36358179db2ed231dd101c48dd298e7b0", + "sha256:0e3a41aad5919613483aad9ebd53336905cab1bd6788afd3995c2a972d89d795", + "sha256:306788fd019bb90e9cbb83d3f3c6becad1c048dd432af24f8320cf38ac085684", + "sha256:39ebd8e120cb77a06ee3d5fc26f9732670d1c397d7cd3acf02f6f62693b89b80", + "sha256:411fdd9f4203afd93b056c0868c8f9e5e16813e765de962f27e4e5798356a052", + "sha256:4822327b35cb032ff16af3bec27f73985448f08e874146b5b101e0e558b613dd", + "sha256:52f8b9fcf3c5e427d51bbab1fb92b575a9a9235d516f175b24712bcd4b5be917", + "sha256:53c8edd3b83a4ddba3d8c506f1359401e7770b30f2188f15c17a338adf5a14db", + "sha256:555a498999c44f5287cc95500486cd0d4f021af9162982cbe504d4cb388f73b5", + "sha256:59fc88bc13e30f25167e807b8cad3c41b7218ef4473a20c86fd98a7968733083", + "sha256:5a559aab40c716de80c7212295d0dc96bc1b6c719371c20dd18c5187c3155518", + "sha256:5de1e9335e2569974e20df0ce31493d315a830d7987e71a24a2a335a8d8459d3", + "sha256:6630d8d943644ea62132789940ca97d05fac83f73186eaf0930ffa715fbdab6b", + "sha256:73a10939dc345460ca0655356a470dd3de9759919186a82383c87b6eb315faf2", + "sha256:7856ea39059d75f822ff0df3a51ea6d76307c897048bdec3aad1377e4e9dca20", + "sha256:877ee5478fd78e100362aed56db47ccc5f23f6e7bb035a8896855f4c3e49bc9b", + "sha256:920a734fe3d311ca01883b4a19aa386c97b82b69fbc023458899cff0a0d621b9", + "sha256:923f9084d7e1d31b5f74c92396b05b18921ed01ee5350402b561a79dce3ea48d", + "sha256:a0d2df4227f645a879010461df2cea6b7e3fb5a97d7eafa210f7fb60345af9e8", + "sha256:a2738ba1ee544d6f294278cfb6de2dc1f9a737a780469b5366e662a218f806c3", + "sha256:a42eaaae772f14a5194f181740a67bfd48e8806394b8c67aa4399e09d0d6b5db", + "sha256:ab2b1a89d2bc7647622e9eaf06128a5b5451dccf7c242deaa31420b055716481", + "sha256:ab9ef0187d6c62b09dec83a84a3b94f71f9690784c84fd762fb3cf2d2b44c914", + "sha256:adf1a0d272633b21d645dd6e02e3293429c1141c7d65a58e4cbcd592d53b8e01", + "sha256:b104b6b1827d6a22483c469e3983a204bcf9c6bf7544bf90362c4654ebc2edf3", + "sha256:bc698580216050b5f4a34d2cdd2838b429c53314f1c4835fab7338200a8396f2", + "sha256:cdf7b83f04a313a21afb1f8730fe4dd09577fefc53bbdfececf78b2006f4268e", + "sha256:d5191d53afbe5b6059895fa7f58223d3751c42b8101fb3ce767e1a0b1a1d8f87", + "sha256:d75314b00825d70e1e34b07396e23f47ed1d4feedc0122748f9f6bd31a544840", + "sha256:e4d64304acf79766e650f7acb81d263a3ea6e2d0d04c5172b7189180ff2c023c", + "sha256:ec2ae1f398e5aca655b7084392d23e80efb31f7a660d2eecf569fb9f79b3fb94", + "sha256:eff095a5aac7011fdb51a2c82a8fae9ec5211577f4b764e1e59cfa27ceeb1b59", + "sha256:f1eda5cae434282712e40b42aaf590b773382afc3642786ac3ed39053973f61f", + "sha256:f217850ac0e046ede611312703423767ca032a7b952b5257efac963942c055de", + "sha256:f50d3a822947572496ea922ee7825becd8e3ae6fbd2400cd8236b7d64b17f285", + "sha256:fc294de50941d3da66a09dca06e206297709332050973eca17040278cb0918ff", + "sha256:ff9832434a9193fbd716fbe05f9276484e18d26cc4cf850853594bb322807ac3" ], "markers": "python_version >= '3.7'", - "version": "==6.4.2" + "version": "==6.4.3" }, "docutils": { "hashes": [ @@ -1231,19 +1217,19 @@ }, "faker": { "hashes": [ - "sha256:8e94a749d2f3d9b367f61eb33be6a534f0a2d305c54e912ee6618370e3278db7", - "sha256:a126fa66f54e65a67f913dcc698c9d023def7277882536bde2968fcac701bfd5" + "sha256:0c7d283a96c49af64fe319f70d2b68927873c9173e922f8eda6001e7757cb63b", + "sha256:f1558ecb1770d8c871ea01cc2edc7b5e86148b0fa0466731f0e1e8953165d179" ], "markers": "python_version >= '3.6'", - "version": "==13.15.0" + "version": "==14.0.0" }, "flake8": { "hashes": [ - "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", - "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db", + "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248" ], "index": "pypi", - "version": "==4.0.1" + "version": "==5.0.4" }, "flake8-docstrings": { "hashes": [ @@ -1301,17 +1287,17 @@ }, "jmespath": { "hashes": [ - "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", - "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.10.0" + "markers": "python_version >= '3.7'", + "version": "==1.0.1" }, "localstack-client": { "hashes": [ - "sha256:f42a8cb0b16bab56e447058f06b4d583814d03be9687e8c7dac6cf35b7c2210b" + "sha256:3121b3daabbdd06da18c73b243a93106ac58b6e5c93ee2a5adbe1eb844fc4b96" ], - "version": "==1.35" + "version": "==1.36" }, "markdown": { "hashes": [ @@ -1369,10 +1355,11 @@ }, "mccabe": { "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" ], - "version": "==0.6.1" + "markers": "python_version >= '3.6'", + "version": "==0.7.0" }, "mergedeep": { "hashes": [ @@ -1384,11 +1371,11 @@ }, "mkdocs": { "hashes": [ - "sha256:26bd2b03d739ac57a3e6eed0b7bcc86168703b719c27b99ad6ca91dc439aacde", - "sha256:b504405b04da38795fec9b2e5e28f6aa3a73bb0960cb6d5d27ead28952bd35ea" + "sha256:a41a2ff25ce3bbacc953f9844ba07d106233cd76c88bac1f59cb1564ac0d87ed", + "sha256:fda92466393127d2da830bc6edc3a625a14b436316d1caf347690648e774c4f0" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.3.1" }, "packaging": { "hashes": [ @@ -1434,11 +1421,11 @@ }, "pycodestyle": { "hashes": [ - "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", - "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785", + "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.8.0" + "markers": "python_version >= '3.6'", + "version": "==2.9.1" }, "pydocstyle": { "hashes": [ @@ -1450,11 +1437,11 @@ }, "pyflakes": { "hashes": [ - "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", - "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2", + "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.0" + "markers": "python_version >= '3.6'", + "version": "==2.5.0" }, "pygraphviz": { "hashes": [ @@ -1572,10 +1559,11 @@ }, "s3transfer": { "hashes": [ - "sha256:9b3752887a2880690ce628bc263d6d13a3864083aeacff4890c1c9839a5eb0bc", - "sha256:cb022f4b16551edebbb31a377d3f09600dbada7363d8c5db7976e7f47732e1b2" + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" ], - "version": "==0.4.2" + "markers": "python_version >= '3.7'", + "version": "==0.6.0" }, "six": { "hashes": [ @@ -1610,11 +1598,11 @@ }, "urllib3": { "hashes": [ - "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", - "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", - "version": "==1.26.10" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.11" }, "watchdog": { "hashes": [ diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index e4b5ed9979..a5e6c60557 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,4 +1,5 @@ -"""Celery task configurations.""" +"""Tasks class for upload configurations.""" + from __future__ import absolute_import import hashlib @@ -13,10 +14,6 @@ logger = logging.getLogger(__name__) -server_address = settings.SERVER_ADDRESS -local_key = settings.LOCAL_KEY -username = settings.USERNAME - def write_key_to_file(private_key): """Paramiko require the key in file object format.""" @@ -27,7 +24,12 @@ def write_key_to_file(private_key): @shared_task -def upload(data_file_pk): +def upload(data_file_pk, + server_address=settings.SERVER_ADDRESS, + local_key=settings.LOCAL_KEY, + username=settings.USERNAME, + port=22 + ): """ Upload to SFTP server. @@ -42,21 +44,35 @@ def upload(data_file_pk): file_name=data_file.create_filename(), ) + def create_dir(directory_name): + """The code snippet to create directory""" + try: + sftp.chdir(directory_name) # Test if remote_path exists + except IOError: + sftp.mkdir(directory_name) # Create remote_path + sftp.chdir(directory_name) + try: - logger.info(str(file_transfer_record.__dict__)) + # Create directory names for ACF titan destination = str(data_file.create_filename()) - logger.info(destination) today_date = datetime.datetime.today() upper_directory_name = today_date.strftime('%Y%m%d') lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) + + # Paramiko SSH connection requires private key as file + temp_key_file = write_key_to_file(local_key) + os.chmod(temp_key_file, 0o666) + + # Create SFTP/SSH connection transport = paramiko.SSHClient() transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - temp_key_file = write_key_to_file(local_key) - transport.connect(server_address, key_filename=temp_key_file, username=username) + transport.connect(server_address, key_filename=temp_key_file, username=username, port=port) os.remove(temp_key_file) sftp = transport.open_sftp() - transport.exec_command('mkdir -p ' + upper_directory_name + - '/' + lower_directory_name) + + # Create remote directory + create_dir(upper_directory_name) + create_dir(lower_directory_name) f = data_file.file.read() with open(destination, 'wb') as f1: f1.write(f) @@ -64,16 +80,17 @@ def upload(data_file_pk): file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() f1.close() - # temp file can be the file object - sftp.put(destination, upper_directory_name + '/' + lower_directory_name + '/' + destination) + sftp.put(destination, destination) os.remove(destination) logger.info('File {} has been successfully uploaded to {}'.format(destination, server_address)) file_transfer_record.result = LegacyFileTransfer.Result.COMPLETED file_transfer_record.save() + transport.close() return True except Exception as e: logger.error('Failed to upload {} with error:{}'.format(destination, e)) file_transfer_record.file_size = 0 file_transfer_record.result = LegacyFileTransfer.Result.ERROR file_transfer_record.save() + transport.close() return False diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index fb9bf7fe42..83c56c70b3 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -1 +1,99 @@ -"""Integration test(s) for clamav-rest operations.""" +"""Test the upload configurations.""" + +from datetime import datetime + +import pytest +from paramiko import Transport +from paramiko.sftp_client import SFTPClient +from tdpservice.scheduling.tasks import upload +from tdpservice.data_files.test.factories import DataFileFactory +from tdpservice.stts.models import STT + + +@pytest.fixture +def stt_instance(region): + """Return an STT.""" + stt, _ = STT.objects.get_or_create( + name="first", + region=region, + code="AR", + stt_codee=1234, + ) + return stt + +@pytest.fixture +def data_file_instance(): + """Data file fixture instance for testing datafile.""" + return DataFileFactory.create( + created_at=datetime.now(), + ) + + +@pytest.fixture +def sftp_connection_values(sftpserver): + """SFTP connection values for local sftp server.""" + server_address = sftpserver.host + local_key = """-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAsf16LYNvP6vAcxZhSueDCxIpgUbOr4Hj6f37B/nAU+EIbU+3 +i54NXTMmtDjG+xzRNOH110h3o7P3A5B7cFu+IlG0AHzlsBbfO/JAGzcLbtKnxDwn +/C5gFpUNKWqvj9kgqRmrmqPLl7UjfYxZVUSRD8Gh/zIONvTl3C0/EsPn/77e0YH/ +wC7QDyg4rikK9Cksekp9PiskgkeqCIJTk0ETiGGhOODUBOWrhHCZBXAvbbhBe/EV +diU0IUKU869Ci0dKKP3h6YjeDhNwrkNa7jZnXPYPRHdsmWi+RK70dsoZe/MkK2GE +yucjsh8gCHWvltlJ7NQJ0fkrcrEdTp5K9ddVAQIDAQABAoIBAHMMDYIHt8vm/1Ek +gSTCeiCYz70hAI3PHntr0Gv6UgelqCXH5jLXqXm5i2XoYS2FFfDhsV9DFxn4REzI +ghFBK6fROdEq6eglEIwV7Lvqm3g5r3kXDR1i+HcARu6jZ/FJ/mNvFU2yW/Gmgtcu +9bs4w670kPp751Y8e6sSj/dYK8hRCoQlN7B5Ald8bZpVcZA/aVmJLfGvzNBY0D+x +wsTRvZn5Q7XG/BJL0+ZWJoSEi4SPrkENUdlHueRHpwPlzBuiGmksrpYvlnnqwvZD +ld90hxkGgnDCmReTzBRKqSPh3wab+OKzePVrLGDFzhd4V5rSsOJl01uOHac/8QLZ +vHCQEDECgYEA2VHvIkmwu+BpDLXAmD8aiKfzcrEGT32SDVZaNdRcrADXKbtVmSKN +I/UCmtfwNVHZk/FMPqarnfWtqv0B62eovI8qaYNCzNUFCKW3aUcACnnMc9HKlYsf +25jJF2X3JzMv319xYnniy77fr6RsxFw2l/7J3dg0tNeFUvPVMPt8trUCgYEA0at/ +eqzMowG/8dwFubY3t/hlwPT4HuvD1A+/yC5jThU280l7lCwRjk9Eh9024Ihy0XuP ++m6af1IMzhg5fYqrF7sU7Tqxx2m+47J5iQZ+m2n6jSLDslqTcAYKj9XOHMJ3/ANe ++eZKpXxpTstfX7A5Yug9a58PjleMjdm1VpDzKJ0CgYEAhOem58FJZJ0JocxFzNZK +0+hi6nF4+oRBHgcBhIorYsXg0JTQ9KY8yxC8VxZYwUMdXWzkxCwKKMBnRXsWAXGT +sD2eIok0ATEFsxQl5yyUydNTRkG3M12yTgpScQza6g5T6LfmD+Oa4CALjM9x9WSv +vqUDr7jaAv8Len/EkgA7dUECgYEAx5X5/pvZHF5BCgkIhjTXq09QBTLrsft56Tao +t/S4YQ6+xS4w7eZZO99m+/HvGCOrMI/viVOZzBMdz12t9Dx5C1jx3bTeoFWf+X3e +RTqicGycrZbnNLMV4DBQA4Vh82yG7KWE1luKuSbJ09CyVBMbPXSXawf5teTPDgSs +ot/OJ90CgYEAvWYQHspz7irepznaDSwnUqbfDPNQr16IN6s3KnR/MGKl6q54uQ80 +2U/k4d3KxHMAFEUVBXSBOwFswoz0ZgR9bzMUDsoSwtnfXoyFvG1GiW20diNpjPIg +toSg3kO4adt6TdPPc1hRlbKah+Ft20qVhvaPAmQDJD3Ow8we7UAZIas= +-----END RSA PRIVATE KEY-----""" + username = "user" + port = sftpserver.port + + return { + 'server_address': server_address, + 'username': username, + 'local_key': local_key, + 'port': port + } + +@pytest.fixture(scope="session") +def sftpclient(sftpserver): + """SFTP client fixture for testing sftp.""" + transport = Transport((sftpserver.host, sftpserver.port)) + transport.connect(username="a", password="b") + sftpclient = SFTPClient.from_transport(transport) + yield sftpclient + sftpclient.close() + transport.close() + +@pytest.mark.django_db +def test_new_data_file(sftpserver, data_file_instance, sftp_connection_values, sftpclient): + """Datafile object for testing the file.""" + data_file_instance.save() + with sftpserver.serve_content({'a_dir': {'somefile.txt': "File content"}}): + upload(data_file_instance.pk, + server_address=sftp_connection_values['server_address'], + local_key=sftp_connection_values['local_key'], + username=sftp_connection_values['username'], + port=sftp_connection_values['port']) + today_date = datetime.today() + upper_directory_name = today_date.strftime('%Y%m%d') + lower_directory_name = today_date.strftime(str(data_file_instance.year) + + '-' + + str(data_file_instance.quarter)) + assert sftpclient.listdir(upper_directory_name+'/'+lower_directory_name)[0] == \ + data_file_instance.create_filename() From fc7f9baf0380badfa987e25a28f2ca4aeb50aa13 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 15 Aug 2022 11:29:15 -0400 Subject: [PATCH 049/113] 1133: linting --- tdrs-backend/tdpservice/scheduling/tasks.py | 2 +- tdrs-backend/tdpservice/scheduling/test/test_file_upload.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index a5e6c60557..e39b160be4 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,4 +1,4 @@ -"""Tasks class for upload configurations.""" +"""Task class for upload configurations.""" from __future__ import absolute_import diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index 83c56c70b3..9d5ce96c34 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -1,4 +1,4 @@ -"""Test the upload configurations.""" +"""Test class for the upload configurations.""" from datetime import datetime From 20bb9a3469d31af9c7c2015672c9ef30153b9c12 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 15 Aug 2022 12:17:15 -0400 Subject: [PATCH 050/113] 1133: linting --- tdrs-backend/tdpservice/scheduling/tasks.py | 6 +++--- tdrs-backend/tdpservice/scheduling/test/test_file_upload.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index e39b160be4..26ae49a8d8 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,6 +1,6 @@ -"""Task class for upload configurations.""" +"""schedule tasks.""" -from __future__ import absolute_import +# The tasks import hashlib import os @@ -45,7 +45,7 @@ def upload(data_file_pk, ) def create_dir(directory_name): - """The code snippet to create directory""" + """Code snippet to create directory.""" try: sftp.chdir(directory_name) # Test if remote_path exists except IOError: diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index 9d5ce96c34..9674b57b4d 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -1,4 +1,4 @@ -"""Test class for the upload configurations.""" +"""Scheduling tests.""" from datetime import datetime @@ -23,7 +23,7 @@ def stt_instance(region): @pytest.fixture def data_file_instance(): - """Data file fixture instance for testing datafile.""" + """Prepare data file fixture instance for testing datafile.""" return DataFileFactory.create( created_at=datetime.now(), ) From 8e2fe88ee45f23a25f0a77b95dcb96996ea42f23 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 08:46:01 -0400 Subject: [PATCH 051/113] 1133: changes for passing the test --- tdrs-backend/tdpservice/data_files/views.py | 7 ++++++- tdrs-backend/tdpservice/scheduling/tasks.py | 8 ++++---- tdrs-backend/tdpservice/settings/common.py | 9 ++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index f7d36066f1..d0d850cd1a 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -3,6 +3,7 @@ from django.http import StreamingHttpResponse from django_filters import rest_framework as filters +from django.conf import settings from drf_yasg.openapi import Parameter from drf_yasg.utils import swagger_auto_schema from rest_framework.parsers import MultiPartParser @@ -60,7 +61,11 @@ def create(self, request, *args, **kwargs): # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: tasks.upload.delay( - data_file_pk=response.data.get('id') + data_file_pk=response.data.get('id'), + server_address=settings.ACFTITAN_SERVER_ADDRESS, + local_key=settings.ACFTITAN_LOCAL_KEY, + username=settings.ACFTITAN_USERNAME, + port=22 ) return response diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 26ae49a8d8..f0171432ca 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -25,9 +25,9 @@ def write_key_to_file(private_key): @shared_task def upload(data_file_pk, - server_address=settings.SERVER_ADDRESS, - local_key=settings.LOCAL_KEY, - username=settings.USERNAME, + server_address=settings.ACFTITAN_SERVER_ADDRESS, + local_key=settings.ACFTITAN_LOCAL_KEY, + username=settings.ACFTITAN_USERNAME, port=22 ): """ @@ -61,7 +61,7 @@ def create_dir(directory_name): # Paramiko SSH connection requires private key as file temp_key_file = write_key_to_file(local_key) - os.chmod(temp_key_file, 0o666) + os.chmod(temp_key_file, 0o600) # Create SFTP/SSH connection transport = paramiko.SSHClient() diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 38790087ad..71402ab989 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -400,9 +400,12 @@ class Common(Configuration): ) # sftp config - SERVER_ADDRESS = os.getenv('ACFTITAN_HOST', '') - LOCAL_KEY = os.getenv('ACFTITAN_KEY', '') - USERNAME = os.getenv('ACFTITAN_USERNAME', '') + ACFTITAN_SERVER_ADDRESS = os.getenv('ACFTITAN_HOST', '') + ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '') + ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') + + # sftp test key + ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', '') # CELERY CONFIG REDIS_URI = os.getenv( From a759ed93121227aefa4629282ef590981d0a1ab4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 09:41:34 -0400 Subject: [PATCH 052/113] 1133: added import --- tdrs-backend/tdpservice/scheduling/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index f0171432ca..470a4e8652 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -1,5 +1,5 @@ """schedule tasks.""" - +from __future__ import absolute_import # The tasks import hashlib From 0a00a4fa5931754ff92097cbd271fd13fdc0f05e Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 10:25:47 -0400 Subject: [PATCH 053/113] 1133: add logging --- tdrs-backend/tdpservice/data_files/views.py | 3 ++- tdrs-backend/tdpservice/scheduling/tasks.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index d0d850cd1a..9e6432ae71 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -57,7 +57,8 @@ class DataFileViewSet(ModelViewSet): def create(self, request, *args, **kwargs): """Override create to upload in case of successful scan.""" response = super().create(request, *args, **kwargs) - + logger.info('_________ in create') + logger.info(str(response.status_code)) # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: tasks.upload.delay( diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 470a4e8652..0c65f7ef84 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -30,6 +30,7 @@ def upload(data_file_pk, username=settings.ACFTITAN_USERNAME, port=22 ): + logger.info('__________________in upload') """ Upload to SFTP server. From 0584942bfbabefdd6fe1f48428421d8e783a8fa6 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 10:31:51 -0400 Subject: [PATCH 054/113] 1133: add logging --- tdrs-backend/tdpservice/scheduling/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 0c65f7ef84..b4a1a51a6e 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -30,13 +30,13 @@ def upload(data_file_pk, username=settings.ACFTITAN_USERNAME, port=22 ): - logger.info('__________________in upload') """ Upload to SFTP server. This task uploads the file in DataFile object with pk = data_file_pk to sftp server as defined in Settings file """ + logger.info('__________________in upload') # Upload file data_file = DataFile.objects.get(id=data_file_pk) file_transfer_record = LegacyFileTransfer( From f7eaeb5e69357327938b17b29e665096061c7a11 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 10:55:50 -0400 Subject: [PATCH 055/113] 1133: add logging --- tdrs-backend/tdpservice/data_files/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 9e6432ae71..4557cc99bf 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -57,8 +57,8 @@ class DataFileViewSet(ModelViewSet): def create(self, request, *args, **kwargs): """Override create to upload in case of successful scan.""" response = super().create(request, *args, **kwargs) - logger.info('_________ in create') - logger.info(str(response.status_code)) + logger.debug('_________ in create') + logger.debug(str(response.status_code)) # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: tasks.upload.delay( From e8e1e7410d3290c24b1f1c2a0598e94ed6fd24f4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 11:15:23 -0400 Subject: [PATCH 056/113] 1133: add logging --- tdrs-backend/tdpservice/data_files/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index c8492e05cb..3ae00972c8 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -40,6 +40,7 @@ class Meta: def create(self, validated_data): """Create a new entry with a new version number.""" + logger.debug('_________ in serializer create') data_file = DataFile.create_new_version(validated_data) # Determine the matching ClamAVFileScan for this DataFile. From f93005e084c24e27b542f2210b0d1a869ca07cde Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 11:21:03 -0400 Subject: [PATCH 057/113] 1133: add logging --- tdrs-backend/tdpservice/data_files/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index 3ae00972c8..4b16293398 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -1,5 +1,5 @@ """Serialize stt data.""" - +import logging from rest_framework import serializers from tdpservice.data_files.errors import ImmutabilityError @@ -11,7 +11,7 @@ from tdpservice.security.models import ClamAVFileScan from tdpservice.stts.models import STT from tdpservice.users.models import User - +logger = logging.getLogger() class DataFileSerializer(serializers.ModelSerializer): """Serializer for Data files.""" From f29e7a9c22aedd303facb573313a2493e7e15c84 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 11:37:36 -0400 Subject: [PATCH 058/113] 1133: add logging --- tdrs-backend/tdpservice/data_files/serializers.py | 2 +- tdrs-backend/tdpservice/data_files/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index 4b16293398..da7d3ab3d8 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -11,7 +11,7 @@ from tdpservice.security.models import ClamAVFileScan from tdpservice.stts.models import STT from tdpservice.users.models import User -logger = logging.getLogger() +logger = logging.getLogger(__name__) class DataFileSerializer(serializers.ModelSerializer): """Serializer for Data files.""" diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 4557cc99bf..62f5d7485a 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -20,7 +20,7 @@ from tdpservice.users.permissions import DataFilePermissions from tdpservice.scheduling import tasks -logger = logging.getLogger() +logger = logging.getLogger(__name__) class DataFileFilter(filters.FilterSet): From c737ed7859a1f74ed3d68fbdae085b9db8f3db79 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 11:53:44 -0400 Subject: [PATCH 059/113] 1133: add logging --- tdrs-backend/tdpservice/data_files/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 62f5d7485a..a93dc15da0 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -59,8 +59,10 @@ def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) logger.debug('_________ in create') logger.debug(str(response.status_code)) + logger.debug(settings.ACFTITAN_LOCAL_KEY) # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: + logger.debug('----- submitting task -----') tasks.upload.delay( data_file_pk=response.data.get('id'), server_address=settings.ACFTITAN_SERVER_ADDRESS, From 98ba904ef741ac750fc8e30bea3752fec72018b2 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 12:52:04 -0400 Subject: [PATCH 060/113] 1133: add logging to celery.py --- tdrs-backend/tdpservice/settings/celery.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index d66f30f89c..b5a96a1785 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -5,8 +5,10 @@ import configurations # Set the default Django settings module for the 'celery' program. -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.local") -os.environ.setdefault("DJANGO_CONFIGURATION", "Local") +# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.local") +# os.environ.setdefault("DJANGO_CONFIGURATION", "Local") +import logging +logger = logging.getLogger(__name__) configurations.setup() @@ -16,6 +18,6 @@ # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') - +logger.debug('+++++++++++++++++++++ ' + str(app.__dict__)) # Load task modules from all registered Django apps. app.autodiscover_tasks() From c4a899cea1407c59a8743273f58ef45448c5509c Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 12:57:40 -0400 Subject: [PATCH 061/113] 1133: change celery start step --- tdrs-backend/gunicorn_start.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 3458834767..cf259a8cee 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -19,11 +19,6 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -l info & -sleep 5 -celery -A tdpservice.settings --broker=$REDIS_URI flower & -celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & - echo "Starting Gunicorn" if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then gunicorn_params="--bind 0.0.0.0:8080 --timeout 10 --workers 3 --reload --log-level $LOGGING_LEVEL" @@ -34,3 +29,8 @@ fi gunicorn_cmd="gunicorn tdpservice.wsgi:application $gunicorn_params" exec $gunicorn_cmd + +celery -A tdpservice.settings worker -l debug & +sleep 5 +celery -A tdpservice.settings --broker=$REDIS_URI flower & +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & From 2f19dc5527cb3be71aa55ca05d5bad00648299ff Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 13:03:36 -0400 Subject: [PATCH 062/113] 1133: change celery.py for linting --- tdrs-backend/tdpservice/settings/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index b5a96a1785..4880973aad 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,6 +1,6 @@ """Celey configuration file.""" from __future__ import absolute_import -import os +#import os from celery import Celery import configurations From 21a14182b87d013758d9de43f9953728d7030d45 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 13:09:05 -0400 Subject: [PATCH 063/113] 1133: linting --- tdrs-backend/tdpservice/settings/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 4880973aad..d9b39ba027 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,6 +1,6 @@ """Celey configuration file.""" from __future__ import absolute_import -#import os +# import os from celery import Celery import configurations From 878f6352284fe2f33eba60fe4c6a74df6a319b7d Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 13:30:31 -0400 Subject: [PATCH 064/113] 1133: update config --- tdrs-backend/gunicorn_start.sh | 10 +++++----- tdrs-backend/tdpservice/settings/celery.py | 7 +++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index cf259a8cee..5bc9b089d1 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -19,6 +19,11 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput +celery -A tdpservice.settings worker -l debug & +sleep 5 +celery -A tdpservice.settings --broker=$REDIS_URI flower & +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & + echo "Starting Gunicorn" if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then gunicorn_params="--bind 0.0.0.0:8080 --timeout 10 --workers 3 --reload --log-level $LOGGING_LEVEL" @@ -29,8 +34,3 @@ fi gunicorn_cmd="gunicorn tdpservice.wsgi:application $gunicorn_params" exec $gunicorn_cmd - -celery -A tdpservice.settings worker -l debug & -sleep 5 -celery -A tdpservice.settings --broker=$REDIS_URI flower & -celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index d9b39ba027..71b8bba6ac 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,12 +1,12 @@ """Celey configuration file.""" from __future__ import absolute_import -# import os +import os from celery import Celery import configurations # Set the default Django settings module for the 'celery' program. -# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.local") -# os.environ.setdefault("DJANGO_CONFIGURATION", "Local") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.cloudgov") +os.environ.setdefault("DJANGO_CONFIGURATION", "CloudGov") import logging logger = logging.getLogger(__name__) @@ -18,6 +18,5 @@ # - namespace='CELERY' means all celery-related configuration keys # should have a `CELERY_` prefix. app.config_from_object('django.conf:settings', namespace='CELERY') -logger.debug('+++++++++++++++++++++ ' + str(app.__dict__)) # Load task modules from all registered Django apps. app.autodiscover_tasks() From 966b2c6c5bf2b5a2e1d66de0ee96697a4b9fc72a Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 13:47:29 -0400 Subject: [PATCH 065/113] 1133: linting --- tdrs-backend/tdpservice/settings/celery.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 71b8bba6ac..517dc2ed3a 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,14 +1,13 @@ """Celey configuration file.""" from __future__ import absolute_import import os -from celery import Celery import configurations +from celery import Celery + # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tdpservice.settings.cloudgov") os.environ.setdefault("DJANGO_CONFIGURATION", "CloudGov") -import logging -logger = logging.getLogger(__name__) configurations.setup() From 899b9031895cd09457b0eb4cc57b4ec36888125c Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 15:15:41 -0400 Subject: [PATCH 066/113] 1133: removed scheduling --- tdrs-backend/gunicorn_start.sh | 6 +++--- tdrs-backend/tdpservice/settings/common.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 5bc9b089d1..ac75a640ea 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -20,9 +20,9 @@ python manage.py populate_stts python manage.py collectstatic --noinput celery -A tdpservice.settings worker -l debug & -sleep 5 -celery -A tdpservice.settings --broker=$REDIS_URI flower & -celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & +# sleep 5 +# celery -A tdpservice.settings --broker=$REDIS_URI flower & +# celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & echo "Starting Gunicorn" if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 71402ab989..7db004b4b0 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -47,7 +47,7 @@ class Common(Configuration): "corsheaders", "django_extensions", "drf_yasg", - "django_celery_beat", + #"django_celery_beat", "storages", # Local apps "tdpservice.core.apps.CoreConfig", From a8a852535746f1600c5d32b8d9d679ffdbda5c6d Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 15:27:06 -0400 Subject: [PATCH 067/113] 1133: permissions --- tdrs-backend/tdpservice/users/test/test_permissions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 1b546934ea..6899a0ab8b 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -69,6 +69,7 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'data_files.view_legacyfiletransfer', 'data_files.add_legacyfiletransfer', 'data_files.change_legacyfiletransfer', + """ 'django_celery_beat.add_clockedschedule', 'django_celery_beat.add_crontabschedule', 'django_celery_beat.add_intervalschedule', @@ -86,7 +87,8 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'django_celery_beat.view_intervalschedule', 'django_celery_beat.view_periodictask', 'django_celery_beat.view_periodictasks', - 'django_celery_beat.view_solarschedule', + 'django_celery_beat.view_solarschedule', + """ } group_permissions = ofa_system_admin.get_group_permissions() assert group_permissions == expected_permissions From 5913030e6acd8050a6538daeb271f1f220e75a09 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 15:33:32 -0400 Subject: [PATCH 068/113] 1133: permissions --- .../tdpservice/users/test/test_permissions.py | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 6899a0ab8b..31aa451f13 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -68,27 +68,7 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'users.view_user', 'data_files.view_legacyfiletransfer', 'data_files.add_legacyfiletransfer', - 'data_files.change_legacyfiletransfer', - """ - 'django_celery_beat.add_clockedschedule', - 'django_celery_beat.add_crontabschedule', - 'django_celery_beat.add_intervalschedule', - 'django_celery_beat.add_periodictask', - 'django_celery_beat.add_periodictasks', - 'django_celery_beat.add_solarschedule', - 'django_celery_beat.change_clockedschedule', - 'django_celery_beat.change_crontabschedule', - 'django_celery_beat.change_intervalschedule', - 'django_celery_beat.change_periodictask', - 'django_celery_beat.change_periodictasks', - 'django_celery_beat.change_solarschedule', - 'django_celery_beat.view_clockedschedule', - 'django_celery_beat.view_crontabschedule', - 'django_celery_beat.view_intervalschedule', - 'django_celery_beat.view_periodictask', - 'django_celery_beat.view_periodictasks', - 'django_celery_beat.view_solarschedule', - """ + 'data_files.change_legacyfiletransfer' } group_permissions = ofa_system_admin.get_group_permissions() assert group_permissions == expected_permissions From 15343de1203d7fbfcc5c2471afbf6240ce3b8ee3 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 18:40:06 -0400 Subject: [PATCH 069/113] 1133: changed redis and celery memory --- tdrs-backend/gunicorn_start.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index ac75a640ea..cec8ec3944 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -9,7 +9,7 @@ if [[ "$REDIS_SERVER_LOCAL" = "TRUE" ]]; then else echo "Run redis server locally" export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:/home/vcap/deps/1/lib:$LD_LIBRARY_PATH - ( cd /home/vcap/deps/0/bin/; ./redis-server &) + ( cd /home/vcap/deps/0/bin/; echo 'maxmemory 64mb' | ./redis-server &) fi # @@ -19,7 +19,7 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -l debug & +celery -A tdpservice.settings worker -c 4 --max-memory-per-child 10000 # sleep 5 # celery -A tdpservice.settings --broker=$REDIS_URI flower & # celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & From 6b9d748694ec0d3eb173ddc4b2c82b83c0c820bd Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 17 Aug 2022 19:08:02 -0400 Subject: [PATCH 070/113] 1133: changed redis and celery memory --- tdrs-backend/gunicorn_start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index cec8ec3944..6e668911cc 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -19,7 +19,7 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -c 4 --max-memory-per-child 10000 +celery -A tdpservice.settings worker -c 4 --max-memory-per-child 2000 # sleep 5 # celery -A tdpservice.settings --broker=$REDIS_URI flower & # celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & From ebd50496fe7e6564cbaaf5306cc1ce023a293bcd Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 07:57:34 -0400 Subject: [PATCH 071/113] 1133: changed redis and celery memory and added commenting --- tdrs-backend/__init__.py | 0 tdrs-backend/gunicorn_start.sh | 2 +- tdrs-backend/redis.conf | 9 ++++ .../scheduling/test/test_file_upload.py | 44 +++++++------------ 4 files changed, 25 insertions(+), 30 deletions(-) delete mode 100644 tdrs-backend/__init__.py create mode 100644 tdrs-backend/redis.conf diff --git a/tdrs-backend/__init__.py b/tdrs-backend/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 6e668911cc..12b8e4393f 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -9,7 +9,7 @@ if [[ "$REDIS_SERVER_LOCAL" = "TRUE" ]]; then else echo "Run redis server locally" export LD_LIBRARY_PATH=/home/vcap/deps/0/lib/:/home/vcap/deps/1/lib:$LD_LIBRARY_PATH - ( cd /home/vcap/deps/0/bin/; echo 'maxmemory 64mb' | ./redis-server &) + ( cd /home/vcap/deps/0/bin/; ./redis-server /home/vcap/app/redis.conf &) fi # diff --git a/tdrs-backend/redis.conf b/tdrs-backend/redis.conf new file mode 100644 index 0000000000..f322213f34 --- /dev/null +++ b/tdrs-backend/redis.conf @@ -0,0 +1,9 @@ +#############NETWORK################ +bind 0.0.0.0 // listen on all addresses +port 6379 // default bind port +timeout 3600s // close connection after client is idle for N seconds +tcp-keepalive 60 + +##############MEMORY############# +maxmemory 20mb +maxmemory-policy volatile-lru diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index 9674b57b4d..3a6a915208 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -8,7 +8,12 @@ from tdpservice.scheduling.tasks import upload from tdpservice.data_files.test.factories import DataFileFactory from tdpservice.stts.models import STT +from django.conf import settings +""" +To mock sftp server, pytest_sftpserver (https://github.com/ulope/pytest-sftpserver) is used. +The package provides two main fixtures for testing: sftpserver and sftpclient +""" @pytest.fixture def stt_instance(region): @@ -33,36 +38,9 @@ def data_file_instance(): def sftp_connection_values(sftpserver): """SFTP connection values for local sftp server.""" server_address = sftpserver.host - local_key = """-----BEGIN RSA PRIVATE KEY----- -MIIEpQIBAAKCAQEAsf16LYNvP6vAcxZhSueDCxIpgUbOr4Hj6f37B/nAU+EIbU+3 -i54NXTMmtDjG+xzRNOH110h3o7P3A5B7cFu+IlG0AHzlsBbfO/JAGzcLbtKnxDwn -/C5gFpUNKWqvj9kgqRmrmqPLl7UjfYxZVUSRD8Gh/zIONvTl3C0/EsPn/77e0YH/ -wC7QDyg4rikK9Cksekp9PiskgkeqCIJTk0ETiGGhOODUBOWrhHCZBXAvbbhBe/EV -diU0IUKU869Ci0dKKP3h6YjeDhNwrkNa7jZnXPYPRHdsmWi+RK70dsoZe/MkK2GE -yucjsh8gCHWvltlJ7NQJ0fkrcrEdTp5K9ddVAQIDAQABAoIBAHMMDYIHt8vm/1Ek -gSTCeiCYz70hAI3PHntr0Gv6UgelqCXH5jLXqXm5i2XoYS2FFfDhsV9DFxn4REzI -ghFBK6fROdEq6eglEIwV7Lvqm3g5r3kXDR1i+HcARu6jZ/FJ/mNvFU2yW/Gmgtcu -9bs4w670kPp751Y8e6sSj/dYK8hRCoQlN7B5Ald8bZpVcZA/aVmJLfGvzNBY0D+x -wsTRvZn5Q7XG/BJL0+ZWJoSEi4SPrkENUdlHueRHpwPlzBuiGmksrpYvlnnqwvZD -ld90hxkGgnDCmReTzBRKqSPh3wab+OKzePVrLGDFzhd4V5rSsOJl01uOHac/8QLZ -vHCQEDECgYEA2VHvIkmwu+BpDLXAmD8aiKfzcrEGT32SDVZaNdRcrADXKbtVmSKN -I/UCmtfwNVHZk/FMPqarnfWtqv0B62eovI8qaYNCzNUFCKW3aUcACnnMc9HKlYsf -25jJF2X3JzMv319xYnniy77fr6RsxFw2l/7J3dg0tNeFUvPVMPt8trUCgYEA0at/ -eqzMowG/8dwFubY3t/hlwPT4HuvD1A+/yC5jThU280l7lCwRjk9Eh9024Ihy0XuP -+m6af1IMzhg5fYqrF7sU7Tqxx2m+47J5iQZ+m2n6jSLDslqTcAYKj9XOHMJ3/ANe -+eZKpXxpTstfX7A5Yug9a58PjleMjdm1VpDzKJ0CgYEAhOem58FJZJ0JocxFzNZK -0+hi6nF4+oRBHgcBhIorYsXg0JTQ9KY8yxC8VxZYwUMdXWzkxCwKKMBnRXsWAXGT -sD2eIok0ATEFsxQl5yyUydNTRkG3M12yTgpScQza6g5T6LfmD+Oa4CALjM9x9WSv -vqUDr7jaAv8Len/EkgA7dUECgYEAx5X5/pvZHF5BCgkIhjTXq09QBTLrsft56Tao -t/S4YQ6+xS4w7eZZO99m+/HvGCOrMI/viVOZzBMdz12t9Dx5C1jx3bTeoFWf+X3e -RTqicGycrZbnNLMV4DBQA4Vh82yG7KWE1luKuSbJ09CyVBMbPXSXawf5teTPDgSs -ot/OJ90CgYEAvWYQHspz7irepznaDSwnUqbfDPNQr16IN6s3KnR/MGKl6q54uQ80 -2U/k4d3KxHMAFEUVBXSBOwFswoz0ZgR9bzMUDsoSwtnfXoyFvG1GiW20diNpjPIg -toSg3kO4adt6TdPPc1hRlbKah+Ft20qVhvaPAmQDJD3Ow8we7UAZIas= ------END RSA PRIVATE KEY-----""" + local_key = settings.ACFTITAN_SFTP_PYTEST username = "user" port = sftpserver.port - return { 'server_address': server_address, 'username': username, @@ -70,9 +48,9 @@ def sftp_connection_values(sftpserver): 'port': port } + @pytest.fixture(scope="session") def sftpclient(sftpserver): - """SFTP client fixture for testing sftp.""" transport = Transport((sftpserver.host, sftpserver.port)) transport.connect(username="a", password="b") sftpclient = SFTPClient.from_transport(transport) @@ -80,16 +58,24 @@ def sftpclient(sftpserver): sftpclient.close() transport.close() + @pytest.mark.django_db def test_new_data_file(sftpserver, data_file_instance, sftp_connection_values, sftpclient): """Datafile object for testing the file.""" data_file_instance.save() + + """ + Need .serve_content to keep the communication alive + Here we put a dummy file somefile.txt in a_dir to keep the port open + """ with sftpserver.serve_content({'a_dir': {'somefile.txt': "File content"}}): upload(data_file_instance.pk, server_address=sftp_connection_values['server_address'], local_key=sftp_connection_values['local_key'], username=sftp_connection_values['username'], port=sftp_connection_values['port']) + + # Create directory structure as needed for ACF_TITAN to assert correct directory name today_date = datetime.today() upper_directory_name = today_date.strftime('%Y%m%d') lower_directory_name = today_date.strftime(str(data_file_instance.year) + From b1b190d731618f7219c49b6aa8c640ceab26130e Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 11:08:13 -0400 Subject: [PATCH 072/113] 1133: changed redis and celery memory by adding redis.conf, removed logging and print from testing, linting, changed the private key to env vars --- tdrs-backend/docker-compose.yml | 6 +++-- tdrs-backend/gunicorn_start.sh | 8 +++---- tdrs-backend/redis.conf | 8 +++---- tdrs-backend/tdpservice/data_files/views.py | 8 +------ tdrs-backend/tdpservice/scheduling/tasks.py | 24 ++++++++++++------- .../scheduling/test/test_file_upload.py | 3 ++- tdrs-backend/tdpservice/settings/celery.py | 2 +- tdrs-backend/tdpservice/settings/common.py | 2 +- tdrs-backend/tdpservice/settings/local.py | 6 +++++ .../tdpservice/users/test/test_permissions.py | 20 +++++++++++++++- 10 files changed, 57 insertions(+), 30 deletions(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 43a445d3db..249f84119c 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -68,7 +68,7 @@ services: - AMS_CLIENT_SECRET - AMS_CONFIGURATION_ENDPOINT - ACFTITAN_HOST - - ACFTITAN_KEY + - ACFTITAN_KEY=(`cat LOCAL_SFTP_KEY.PRIV`) - ACFTITAN_USERNAME - REDIS_URI - REDIS_SERVER_LOCAL @@ -90,9 +90,11 @@ services: redis-server: image: "redis:alpine" - command: redis-server + command: redis-server /tdpapp/redis.conf ports: - "6379:6379" + volumes: + - .:/tdpapp volumes: localstack_data: diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 12b8e4393f..e148d6d52d 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -19,10 +19,10 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -c 4 --max-memory-per-child 2000 -# sleep 5 -# celery -A tdpservice.settings --broker=$REDIS_URI flower & -# celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & +celery -A tdpservice.settings worker -c 4 --max-memory-per-child 2000 & +sleep 5 +celery -A tdpservice.settings --broker=$REDIS_URI flower & +celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & echo "Starting Gunicorn" if [[ "$DJANGO_CONFIGURATION" = "Development" || "$DJANGO_CONFIGURATION" = "Local" ]]; then diff --git a/tdrs-backend/redis.conf b/tdrs-backend/redis.conf index f322213f34..8e3ddd81fd 100644 --- a/tdrs-backend/redis.conf +++ b/tdrs-backend/redis.conf @@ -1,9 +1,7 @@ #############NETWORK################ -bind 0.0.0.0 // listen on all addresses -port 6379 // default bind port -timeout 3600s // close connection after client is idle for N seconds +bind 0.0.0.0 +port 6379 tcp-keepalive 60 - -##############MEMORY############# +##############LIMIT MEMORY############# maxmemory 20mb maxmemory-policy volatile-lru diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index a93dc15da0..835c88fbc6 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -1,5 +1,4 @@ """Check if user is authorized.""" -import logging from django.http import StreamingHttpResponse from django_filters import rest_framework as filters @@ -20,8 +19,6 @@ from tdpservice.users.permissions import DataFilePermissions from tdpservice.scheduling import tasks -logger = logging.getLogger(__name__) - class DataFileFilter(filters.FilterSet): """Filters that can be applied to GET requests as query parameters.""" @@ -57,12 +54,9 @@ class DataFileViewSet(ModelViewSet): def create(self, request, *args, **kwargs): """Override create to upload in case of successful scan.""" response = super().create(request, *args, **kwargs) - logger.debug('_________ in create') - logger.debug(str(response.status_code)) - logger.debug(settings.ACFTITAN_LOCAL_KEY) + # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: - logger.debug('----- submitting task -----') tasks.upload.delay( data_file_pk=response.data.get('id'), server_address=settings.ACFTITAN_SERVER_ADDRESS, diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index b4a1a51a6e..17e6ecea93 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -36,7 +36,6 @@ def upload(data_file_pk, This task uploads the file in DataFile object with pk = data_file_pk to sftp server as defined in Settings file """ - logger.info('__________________in upload') # Upload file data_file = DataFile.objects.get(id=data_file_pk) file_transfer_record = LegacyFileTransfer( @@ -45,13 +44,13 @@ def upload(data_file_pk, file_name=data_file.create_filename(), ) - def create_dir(directory_name): - """Code snippet to create directory.""" + def create_dir(directory_name, sftp_server): + """Code snippet to create directory in SFTP server.""" try: - sftp.chdir(directory_name) # Test if remote_path exists + sftp_server.chdir(directory_name) # Test if remote_path exists except IOError: - sftp.mkdir(directory_name) # Create remote_path - sftp.chdir(directory_name) + sftp_server.mkdir(directory_name) # Create remote_path + sftp_server.chdir(directory_name) try: # Create directory names for ACF titan @@ -68,26 +67,35 @@ def create_dir(directory_name): transport = paramiko.SSHClient() transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) transport.connect(server_address, key_filename=temp_key_file, username=username, port=port) + # remove temp key file os.remove(temp_key_file) sftp = transport.open_sftp() # Create remote directory - create_dir(upper_directory_name) - create_dir(lower_directory_name) + create_dir(upper_directory_name, sftp_server=sftp) + create_dir(lower_directory_name, sftp_server=sftp) f = data_file.file.read() + + # Paramiko need local file with open(destination, 'wb') as f1: f1.write(f) file_transfer_record.file_size = f1.tell() file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() f1.close() + # Put the file in SFTP server sftp.put(destination, destination) + + # Delete temp file os.remove(destination) logger.info('File {} has been successfully uploaded to {}'.format(destination, server_address)) + + # Add the log LegacyFileTransfer file_transfer_record.result = LegacyFileTransfer.Result.COMPLETED file_transfer_record.save() transport.close() return True + except Exception as e: logger.error('Failed to upload {} with error:{}'.format(destination, e)) file_transfer_record.file_size = 0 diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index 3a6a915208..ae08e7b0b5 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -12,7 +12,7 @@ """ To mock sftp server, pytest_sftpserver (https://github.com/ulope/pytest-sftpserver) is used. -The package provides two main fixtures for testing: sftpserver and sftpclient +The package provides two main fixtures for testing: sftpserver and sftpclient. """ @pytest.fixture @@ -51,6 +51,7 @@ def sftp_connection_values(sftpserver): @pytest.fixture(scope="session") def sftpclient(sftpserver): + """SFTP client for local sftp server.""" transport = Transport((sftpserver.host, sftpserver.port)) transport.connect(username="a", password="b") sftpclient = SFTPClient.from_transport(transport) diff --git a/tdrs-backend/tdpservice/settings/celery.py b/tdrs-backend/tdpservice/settings/celery.py index 517dc2ed3a..6955ca9cec 100644 --- a/tdrs-backend/tdpservice/settings/celery.py +++ b/tdrs-backend/tdpservice/settings/celery.py @@ -1,4 +1,4 @@ -"""Celey configuration file.""" +"""Celery configuration file.""" from __future__ import absolute_import import os import configurations diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 7db004b4b0..71402ab989 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -47,7 +47,7 @@ class Common(Configuration): "corsheaders", "django_extensions", "drf_yasg", - #"django_celery_beat", + "django_celery_beat", "storages", # Local apps "tdpservice.core.apps.CoreConfig", diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 83be95bd3b..2880e6a590 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -43,3 +43,9 @@ class Local(Common): 'level': 'DEBUG', 'handlers': ['console'] } + + with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: + SFTP_PYTEST_ENV_VAR = f.read() + + # sftp test key + ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) diff --git a/tdrs-backend/tdpservice/users/test/test_permissions.py b/tdrs-backend/tdpservice/users/test/test_permissions.py index 31aa451f13..1b546934ea 100644 --- a/tdrs-backend/tdpservice/users/test/test_permissions.py +++ b/tdrs-backend/tdpservice/users/test/test_permissions.py @@ -68,7 +68,25 @@ def test_ofa_system_admin_permissions(ofa_system_admin): 'users.view_user', 'data_files.view_legacyfiletransfer', 'data_files.add_legacyfiletransfer', - 'data_files.change_legacyfiletransfer' + 'data_files.change_legacyfiletransfer', + 'django_celery_beat.add_clockedschedule', + 'django_celery_beat.add_crontabschedule', + 'django_celery_beat.add_intervalschedule', + 'django_celery_beat.add_periodictask', + 'django_celery_beat.add_periodictasks', + 'django_celery_beat.add_solarschedule', + 'django_celery_beat.change_clockedschedule', + 'django_celery_beat.change_crontabschedule', + 'django_celery_beat.change_intervalschedule', + 'django_celery_beat.change_periodictask', + 'django_celery_beat.change_periodictasks', + 'django_celery_beat.change_solarschedule', + 'django_celery_beat.view_clockedschedule', + 'django_celery_beat.view_crontabschedule', + 'django_celery_beat.view_intervalschedule', + 'django_celery_beat.view_periodictask', + 'django_celery_beat.view_periodictasks', + 'django_celery_beat.view_solarschedule', } group_permissions = ofa_system_admin.get_group_permissions() assert group_permissions == expected_permissions From 30b5162a3ac288cd43acbfbf9a687fa704f20bbd Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 11:33:40 -0400 Subject: [PATCH 073/113] 1133: removed cat ACF key file --- tdrs-backend/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 249f84119c..211b3acc05 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -68,7 +68,7 @@ services: - AMS_CLIENT_SECRET - AMS_CONFIGURATION_ENDPOINT - ACFTITAN_HOST - - ACFTITAN_KEY=(`cat LOCAL_SFTP_KEY.PRIV`) + - ACFTITAN_KEY - ACFTITAN_USERNAME - REDIS_URI - REDIS_SERVER_LOCAL From a2196a442f6843e2fdaf2838e2333e86b3bea01c Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 11:43:40 -0400 Subject: [PATCH 074/113] 1133: settings file: added if statement --- tdrs-backend/tdpservice/settings/local.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 2880e6a590..66d0fa0f2e 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -44,8 +44,9 @@ class Local(Common): 'handlers': ['console'] } - with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: - SFTP_PYTEST_ENV_VAR = f.read() + if os.getenv(REDIS_SERVER_LOCAL,False): + with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: + SFTP_PYTEST_ENV_VAR = f.read() # sftp test key ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) From ea861d205deb5fc42107e2bf0da40b98b5ed5814 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 11:51:32 -0400 Subject: [PATCH 075/113] 1133: settings file linting --- tdrs-backend/tdpservice/settings/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 66d0fa0f2e..e8f2faf676 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -44,7 +44,7 @@ class Local(Common): 'handlers': ['console'] } - if os.getenv(REDIS_SERVER_LOCAL,False): + if os.getenv('REDIS_SERVER_LOCAL', False): with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: SFTP_PYTEST_ENV_VAR = f.read() From 6cd1f1f4eacffeedcc0b72db7a4ff6e0887e9424 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 11:58:08 -0400 Subject: [PATCH 076/113] 1133: settings file linting --- tdrs-backend/tdpservice/settings/local.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index e8f2faf676..0c9553e4a2 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -47,6 +47,8 @@ class Local(Common): if os.getenv('REDIS_SERVER_LOCAL', False): with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: SFTP_PYTEST_ENV_VAR = f.read() + else: + SFTP_PYTEST_ENV_VAR = '' # sftp test key ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) From 075371096997aec4d9794d048ca90167c8ea10d7 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 12:34:10 -0400 Subject: [PATCH 077/113] 1133: changed the pytest_key env var pull up to common.py --- tdrs-backend/tdpservice/settings/common.py | 9 +++++++++ tdrs-backend/tdpservice/settings/local.py | 9 --------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 71402ab989..979625613f 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -419,3 +419,12 @@ class Common(Configuration): CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_TIMEZONE = 'UTC' + + if os.getenv('REDIS_SERVER_LOCAL', False): + with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: + SFTP_PYTEST_ENV_VAR = f.read() + else: + SFTP_PYTEST_ENV_VAR = '' + + # sftp test key + ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 0c9553e4a2..83be95bd3b 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -43,12 +43,3 @@ class Local(Common): 'level': 'DEBUG', 'handlers': ['console'] } - - if os.getenv('REDIS_SERVER_LOCAL', False): - with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: - SFTP_PYTEST_ENV_VAR = f.read() - else: - SFTP_PYTEST_ENV_VAR = '' - - # sftp test key - ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) From 440b32beb1e8ded79c32a42b7a8f13a34e9553d1 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 13:01:55 -0400 Subject: [PATCH 078/113] 1133: common settings file for env vars --- tdrs-backend/tdpservice/settings/common.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 979625613f..8c809a6dcb 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -404,9 +404,6 @@ class Common(Configuration): ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '') ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') - # sftp test key - ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', '') - # CELERY CONFIG REDIS_URI = os.getenv( 'REDIS_URI', From 36ea55779f89f7781f09ee25bbc5bdda47a9ee76 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 13:09:22 -0400 Subject: [PATCH 079/113] 1133: common settings file for env vars --- tdrs-backend/docker-compose.yml | 1 + tdrs-backend/tdpservice/settings/common.py | 12 +++--------- tdrs-backend/tdpservice/settings/local.py | 9 +++++++++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 211b3acc05..7229530d4d 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -72,6 +72,7 @@ services: - ACFTITAN_USERNAME - REDIS_URI - REDIS_SERVER_LOCAL + - ACFTITAN_SFTP_PYTEST volumes: - .:/tdpapp image: tdp diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 8c809a6dcb..71402ab989 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -404,6 +404,9 @@ class Common(Configuration): ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '') ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') + # sftp test key + ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', '') + # CELERY CONFIG REDIS_URI = os.getenv( 'REDIS_URI', @@ -416,12 +419,3 @@ class Common(Configuration): CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_TIMEZONE = 'UTC' - - if os.getenv('REDIS_SERVER_LOCAL', False): - with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: - SFTP_PYTEST_ENV_VAR = f.read() - else: - SFTP_PYTEST_ENV_VAR = '' - - # sftp test key - ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 83be95bd3b..0c9553e4a2 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -43,3 +43,12 @@ class Local(Common): 'level': 'DEBUG', 'handlers': ['console'] } + + if os.getenv('REDIS_SERVER_LOCAL', False): + with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: + SFTP_PYTEST_ENV_VAR = f.read() + else: + SFTP_PYTEST_ENV_VAR = '' + + # sftp test key + ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) From 9cccf7529fa929390767387ad03094a0e892ea3b Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 13:55:59 -0400 Subject: [PATCH 080/113] 1133: Added an env var --- tdrs-backend/tdpservice/settings/local.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 0c9553e4a2..75f77642ca 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -44,7 +44,8 @@ class Local(Common): 'handlers': ['console'] } - if os.getenv('REDIS_SERVER_LOCAL', False): + REDIS_SERVER_LOCAL = os.getenv('REDIS_SERVER_LOCAL', False) + if REDIS_SERVER_LOCAL: with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: SFTP_PYTEST_ENV_VAR = f.read() else: From 5ab9c960a10e2837a70bfc7d0ef60befa7b93d6f Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 14:10:49 -0400 Subject: [PATCH 081/113] 1133: Added an env var --- tdrs-backend/tdpservice/settings/local.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 75f77642ca..814b962ffa 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -45,6 +45,7 @@ class Local(Common): } REDIS_SERVER_LOCAL = os.getenv('REDIS_SERVER_LOCAL', False) + print(REDIS_SERVER_LOCAL) if REDIS_SERVER_LOCAL: with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: SFTP_PYTEST_ENV_VAR = f.read() From 4e52f2b27d1a07dad45d7df55827cf074bfdaca6 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 14:28:17 -0400 Subject: [PATCH 082/113] 1133: Removed default for testing --- tdrs-backend/tdpservice/settings/local.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 814b962ffa..04a31b7f99 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -44,6 +44,7 @@ class Local(Common): 'handlers': ['console'] } + """ REDIS_SERVER_LOCAL = os.getenv('REDIS_SERVER_LOCAL', False) print(REDIS_SERVER_LOCAL) if REDIS_SERVER_LOCAL: @@ -51,6 +52,7 @@ class Local(Common): SFTP_PYTEST_ENV_VAR = f.read() else: SFTP_PYTEST_ENV_VAR = '' + """ # sftp test key - ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', SFTP_PYTEST_ENV_VAR) + ACFTITAN_SFTP_PYTEST = os.getenv("ACFTITAN_SFTP_PYTEST") From e82c2593ae750802a92f0e3fbbab2b0a9ecf17c7 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 14:54:46 -0400 Subject: [PATCH 083/113] 1133: Added end of line replacement for ssh key --- tdrs-backend/tdpservice/settings/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 04a31b7f99..699ef516d0 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -55,4 +55,4 @@ class Local(Common): """ # sftp test key - ACFTITAN_SFTP_PYTEST = os.getenv("ACFTITAN_SFTP_PYTEST") + ACFTITAN_SFTP_PYTEST = os.getenv("ACFTITAN_SFTP_PYTEST").replace('_','\n') From 73c5f51c099e3f8ad71c77576c24dcfb4ea17553 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 15:01:53 -0400 Subject: [PATCH 084/113] 1133: linting --- tdrs-backend/tdpservice/settings/local.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 699ef516d0..3cd86058b1 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -44,15 +44,5 @@ class Local(Common): 'handlers': ['console'] } - """ - REDIS_SERVER_LOCAL = os.getenv('REDIS_SERVER_LOCAL', False) - print(REDIS_SERVER_LOCAL) - if REDIS_SERVER_LOCAL: - with open('LOCAL_SFTP_KEY.PRIV', 'r') as f: - SFTP_PYTEST_ENV_VAR = f.read() - else: - SFTP_PYTEST_ENV_VAR = '' - """ - # sftp test key - ACFTITAN_SFTP_PYTEST = os.getenv("ACFTITAN_SFTP_PYTEST").replace('_','\n') + ACFTITAN_SFTP_PYTEST = os.getenv("ACFTITAN_SFTP_PYTEST").replace('_', '\n') From 38d40156ae937fdc043030fdbcaa2223551e11e3 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 15:37:07 -0400 Subject: [PATCH 085/113] 1133: changing private key and replace '_' with \n --- tdrs-backend/gunicorn_start.sh | 2 +- tdrs-backend/tdpservice/settings/common.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index e148d6d52d..9eb87dfbc2 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -19,7 +19,7 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -c 4 --max-memory-per-child 2000 & +celery -A tdpservice.settings worker -c 1 --max-memory-per-child 2000 & sleep 5 celery -A tdpservice.settings --broker=$REDIS_URI flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 71402ab989..ca260bc5fd 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -401,12 +401,9 @@ class Common(Configuration): # sftp config ACFTITAN_SERVER_ADDRESS = os.getenv('ACFTITAN_HOST', '') - ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '') + ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '').replace('_', '\n') ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') - # sftp test key - ACFTITAN_SFTP_PYTEST = os.getenv('ACFTITAN_SFTP_PYTEST', '') - # CELERY CONFIG REDIS_URI = os.getenv( 'REDIS_URI', From 1c6650e4d814026a98c8a437225a954f411733de Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 17:00:00 -0400 Subject: [PATCH 086/113] 1133: changed to pkey to resolve primary key error --- .../tdpservice/data_files/serializers.py | 1 - tdrs-backend/tdpservice/scheduling/tasks.py | 20 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/serializers.py b/tdrs-backend/tdpservice/data_files/serializers.py index da7d3ab3d8..26331832d4 100644 --- a/tdrs-backend/tdpservice/data_files/serializers.py +++ b/tdrs-backend/tdpservice/data_files/serializers.py @@ -40,7 +40,6 @@ class Meta: def create(self, validated_data): """Create a new entry with a new version number.""" - logger.debug('_________ in serializer create') data_file = DataFile.create_new_version(validated_data) # Determine the matching ClamAVFileScan for this DataFile. diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 17e6ecea93..2422600467 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -59,6 +59,14 @@ def create_dir(directory_name, sftp_server): upper_directory_name = today_date.strftime('%Y%m%d') lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) + # Paramiko need local file + f = data_file.file.read() + with open(destination, 'wb') as f1: + f1.write(f) + file_transfer_record.file_size = f1.tell() + file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() + f1.close() + # Paramiko SSH connection requires private key as file temp_key_file = write_key_to_file(local_key) os.chmod(temp_key_file, 0o600) @@ -66,7 +74,9 @@ def create_dir(directory_name, sftp_server): # Create SFTP/SSH connection transport = paramiko.SSHClient() transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - transport.connect(server_address, key_filename=temp_key_file, username=username, port=port) + + pkey = paramiko.RSAKey.from_private_key_file(temp_key_file) + transport.connect(server_address, pkey=pkey, username=username, port=port) # remove temp key file os.remove(temp_key_file) sftp = transport.open_sftp() @@ -74,14 +84,6 @@ def create_dir(directory_name, sftp_server): # Create remote directory create_dir(upper_directory_name, sftp_server=sftp) create_dir(lower_directory_name, sftp_server=sftp) - f = data_file.file.read() - - # Paramiko need local file - with open(destination, 'wb') as f1: - f1.write(f) - file_transfer_record.file_size = f1.tell() - file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() - f1.close() # Put the file in SFTP server sftp.put(destination, destination) From 741dd80b809fb05f25086bbf899caf2923fb43c3 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 17:30:32 -0400 Subject: [PATCH 087/113] 1133: look for key to false --- tdrs-backend/gunicorn_start.sh | 4 ++-- tdrs-backend/tdpservice/scheduling/tasks.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 9eb87dfbc2..8720fea997 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -19,9 +19,9 @@ python manage.py migrate python manage.py populate_stts python manage.py collectstatic --noinput -celery -A tdpservice.settings worker -c 1 --max-memory-per-child 2000 & +celery -A tdpservice.settings worker -c 1 --max-memory-per-child 5000 & sleep 5 -celery -A tdpservice.settings --broker=$REDIS_URI flower & +# celery -A tdpservice.settings --broker=$REDIS_URI flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & echo "Starting Gunicorn" diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 2422600467..12b2690247 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -76,7 +76,7 @@ def create_dir(directory_name, sftp_server): transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) pkey = paramiko.RSAKey.from_private_key_file(temp_key_file) - transport.connect(server_address, pkey=pkey, username=username, port=port) + transport.connect(server_address, pkey=pkey, username=username, port=port, look_for_keys=False) # remove temp key file os.remove(temp_key_file) sftp = transport.open_sftp() From 448c3428a3421e45a0751c16448212cbe8261e07 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 18 Aug 2022 18:43:20 -0400 Subject: [PATCH 088/113] 1133: disabled private key algorithms --- tdrs-backend/tdpservice/scheduling/tasks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 12b2690247..94c1cd7cea 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -74,9 +74,13 @@ def create_dir(directory_name, sftp_server): # Create SFTP/SSH connection transport = paramiko.SSHClient() transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - pkey = paramiko.RSAKey.from_private_key_file(temp_key_file) - transport.connect(server_address, pkey=pkey, username=username, port=port, look_for_keys=False) + transport.connect(server_address, + pkey=pkey, + username=username, + port=port, + look_for_keys=False, + disabled_algorithms={'pubkeys': ['rsa-sha2-512', 'rsa-sha2-256']}) # remove temp key file os.remove(temp_key_file) sftp = transport.open_sftp() From cebfd507b8ed7488c3373f14d1a45b27d2a754e2 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 19 Aug 2022 08:36:48 -0400 Subject: [PATCH 089/113] 1133: Added comments to settings files --- tdrs-backend/tdpservice/settings/common.py | 9 +++++++-- tdrs-backend/tdpservice/settings/local.py | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index ca260bc5fd..ba5d3ba829 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -399,12 +399,17 @@ class Common(Configuration): '' ) - # sftp config + # ------- SFTP CONFIG ACFTITAN_SERVER_ADDRESS = os.getenv('ACFTITAN_HOST', '') + """ + To be able to fit the PRIVATE KEY in one line as environment variable, we replace the EOL + with an underscore char. + The next line replaces the _ with EOL before using the PRIVATE KEY + """ ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '').replace('_', '\n') ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') - # CELERY CONFIG + # -------- CELERY CONFIG REDIS_URI = os.getenv( 'REDIS_URI', 'redis://redis-server:6379' diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index 3cd86058b1..f919865c19 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -44,5 +44,10 @@ class Local(Common): 'handlers': ['console'] } - # sftp test key + # SFTP TEST KEY + """ + To be able to fit the PRIVATE KEY in one line as environment variable, we replace the EOL + with an underscore char. + The next line replaces the _ with EOL before using the PRIVATE KEY + """ ACFTITAN_SFTP_PYTEST = os.getenv("ACFTITAN_SFTP_PYTEST").replace('_', '\n') From 6d667aa49b7405aecd969ec3a7cebb0ec212faa9 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 19 Aug 2022 09:55:01 -0400 Subject: [PATCH 090/113] 1133: linting --- tdrs-backend/tdpservice/settings/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index f919865c19..02ae7bdd05 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -46,7 +46,7 @@ class Local(Common): # SFTP TEST KEY """ - To be able to fit the PRIVATE KEY in one line as environment variable, we replace the EOL + To be able to fit the PRIVATE KEY in one line as environment variable, we replace the EOL with an underscore char. The next line replaces the _ with EOL before using the PRIVATE KEY """ From a1d8cb033d7e8924cadd23e0543e419eca1c91a2 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 23 Aug 2022 09:04:51 -0400 Subject: [PATCH 091/113] python script to change EOL with _ --- scripts/tools/sshkey_change.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 scripts/tools/sshkey_change.py diff --git a/scripts/tools/sshkey_change.py b/scripts/tools/sshkey_change.py new file mode 100644 index 0000000000..0d2e20168f --- /dev/null +++ b/scripts/tools/sshkey_change.py @@ -0,0 +1,12 @@ +"""Script to change EOL with _ which is required for the key to be used as env variable.""" + +import sys + +if __name__ == "__main__": + key_file_name = sys.argv[1] + with open(key_file_name, 'r') as key_file: + key_string = key_file.read() + key_string = key_string.replace('\n','_') + with open(key_file_name + '_', 'w') as key_file: + key_file.write(key_string) + print('The key is saved in: ', key_file_name + '_') \ No newline at end of file From e8cc8eab6c64abb06e311f5e4f7d239284b90056 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 23 Aug 2022 09:20:28 -0400 Subject: [PATCH 092/113] ENV variable for specific ACF TITAN directory --- tdrs-backend/tdpservice/scheduling/tasks.py | 1 + tdrs-backend/tdpservice/settings/common.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/tasks.py index 94c1cd7cea..12a4a8fa09 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/tasks.py @@ -88,6 +88,7 @@ def create_dir(directory_name, sftp_server): # Create remote directory create_dir(upper_directory_name, sftp_server=sftp) create_dir(lower_directory_name, sftp_server=sftp) + create_dir(settings.ACFTITAN_DIECTORY, sftp_server=sftp) # Put the file in SFTP server sftp.put(destination, destination) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index ba5d3ba829..8f1d0f13f2 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -408,6 +408,7 @@ class Common(Configuration): """ ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '').replace('_', '\n') ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') + ACFTITAN_DIECTORY = os.getenv('ACFTITAN_DIECTORY', '') # -------- CELERY CONFIG REDIS_URI = os.getenv( From 911705718a1993a76ab1c900e420d3ca4a68abdd Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 24 Aug 2022 09:32:30 -0400 Subject: [PATCH 093/113] linting, chaging upload filename, etc --- tdrs-backend/gunicorn_start.sh | 1 + .../scripts}/sshkey_change.py | 4 ++-- tdrs-backend/tdpservice/data_files/views.py | 4 ++-- .../scheduling/{tasks.py => sftp_task.py} | 21 +++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) rename {scripts/tools => tdrs-backend/scripts}/sshkey_change.py (74%) rename tdrs-backend/tdpservice/scheduling/{tasks.py => sftp_task.py} (89%) diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 8720fea997..894dc9f54a 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -21,6 +21,7 @@ python manage.py collectstatic --noinput celery -A tdpservice.settings worker -c 1 --max-memory-per-child 5000 & sleep 5 +# TODO: Uncomment the following line to add flower service when memory limitation is resolved # celery -A tdpservice.settings --broker=$REDIS_URI flower & celery -A tdpservice.settings beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler & diff --git a/scripts/tools/sshkey_change.py b/tdrs-backend/scripts/sshkey_change.py similarity index 74% rename from scripts/tools/sshkey_change.py rename to tdrs-backend/scripts/sshkey_change.py index 0d2e20168f..123a101b32 100644 --- a/scripts/tools/sshkey_change.py +++ b/tdrs-backend/scripts/sshkey_change.py @@ -6,7 +6,7 @@ key_file_name = sys.argv[1] with open(key_file_name, 'r') as key_file: key_string = key_file.read() - key_string = key_string.replace('\n','_') + key_string = key_string.replace('\n', '_') with open(key_file_name + '_', 'w') as key_file: key_file.write(key_string) - print('The key is saved in: ', key_file_name + '_') \ No newline at end of file + print('The key is saved in: ', key_file_name + '_') diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index 835c88fbc6..0800f0fce1 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -17,7 +17,7 @@ from tdpservice.data_files.serializers import DataFileSerializer from tdpservice.data_files.models import DataFile from tdpservice.users.permissions import DataFilePermissions -from tdpservice.scheduling import tasks +from tdpservice.scheduling import sftp_task class DataFileFilter(filters.FilterSet): @@ -57,7 +57,7 @@ def create(self, request, *args, **kwargs): # Upload to ACF-TITAN only if file is passed the virus scan and created if response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_200_OK: - tasks.upload.delay( + sftp_task.upload.delay( data_file_pk=response.data.get('id'), server_address=settings.ACFTITAN_SERVER_ADDRESS, local_key=settings.ACFTITAN_LOCAL_KEY, diff --git a/tdrs-backend/tdpservice/scheduling/tasks.py b/tdrs-backend/tdpservice/scheduling/sftp_task.py similarity index 89% rename from tdrs-backend/tdpservice/scheduling/tasks.py rename to tdrs-backend/tdpservice/scheduling/sftp_task.py index 12a4a8fa09..5aede35e8a 100644 --- a/tdrs-backend/tdpservice/scheduling/tasks.py +++ b/tdrs-backend/tdpservice/scheduling/sftp_task.py @@ -15,14 +15,6 @@ logger = logging.getLogger(__name__) -def write_key_to_file(private_key): - """Paramiko require the key in file object format.""" - with open('temp_key_file', 'w') as f: - f.write(private_key) - f.close() - return 'temp_key_file' - - @shared_task def upload(data_file_pk, server_address=settings.ACFTITAN_SERVER_ADDRESS, @@ -44,6 +36,13 @@ def upload(data_file_pk, file_name=data_file.create_filename(), ) + def write_key_to_file(private_key): + """Paramiko require the key in file object format.""" + with open('temp_key_file', 'w') as f: + f.write(private_key) + f.close() + return 'temp_key_file' + def create_dir(directory_name, sftp_server): """Code snippet to create directory in SFTP server.""" try: @@ -60,11 +59,11 @@ def create_dir(directory_name, sftp_server): lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) # Paramiko need local file - f = data_file.file.read() + paramiko_local_file = data_file.file.read() with open(destination, 'wb') as f1: - f1.write(f) + f1.write(paramiko_local_file) file_transfer_record.file_size = f1.tell() - file_transfer_record.file_shasum = hashlib.sha256(f).hexdigest() + file_transfer_record.file_shasum = hashlib.sha256(paramiko_local_file).hexdigest() f1.close() # Paramiko SSH connection requires private key as file From 49292ba68df67c76e99e72d4c85b2d7ca0212dbe Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 24 Aug 2022 09:43:57 -0400 Subject: [PATCH 094/113] test file reference change --- tdrs-backend/tdpservice/scheduling/test/test_file_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index ae08e7b0b5..55ba6af2ae 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -5,7 +5,7 @@ import pytest from paramiko import Transport from paramiko.sftp_client import SFTPClient -from tdpservice.scheduling.tasks import upload +from tdpservice.scheduling.sftp_task import upload from tdpservice.data_files.test.factories import DataFileFactory from tdpservice.stts.models import STT from django.conf import settings From ed4c9fc0da94e09be94b4b7a6fcf882c664f012f Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 25 Aug 2022 10:49:55 -0400 Subject: [PATCH 095/113] correct typo --- tdrs-backend/tdpservice/scheduling/sftp_task.py | 2 +- tdrs-backend/tdpservice/settings/common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/sftp_task.py b/tdrs-backend/tdpservice/scheduling/sftp_task.py index 5aede35e8a..b65d408cee 100644 --- a/tdrs-backend/tdpservice/scheduling/sftp_task.py +++ b/tdrs-backend/tdpservice/scheduling/sftp_task.py @@ -87,7 +87,7 @@ def create_dir(directory_name, sftp_server): # Create remote directory create_dir(upper_directory_name, sftp_server=sftp) create_dir(lower_directory_name, sftp_server=sftp) - create_dir(settings.ACFTITAN_DIECTORY, sftp_server=sftp) + create_dir(settings.ACFTITAN_DIRECTORY, sftp_server=sftp) # Put the file in SFTP server sftp.put(destination, destination) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 8f1d0f13f2..eb0bd47dcc 100755 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -408,7 +408,7 @@ class Common(Configuration): """ ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '').replace('_', '\n') ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') - ACFTITAN_DIECTORY = os.getenv('ACFTITAN_DIECTORY', '') + ACFTITAN_DIRECTORY = os.getenv('ACFTITAN_DIRECTORY', '') # -------- CELERY CONFIG REDIS_URI = os.getenv( From f1680f04a7ec2d23fbe57962c40f4e24a1af7e0d Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 25 Aug 2022 11:06:03 -0400 Subject: [PATCH 096/113] increase timeout for OWASP --- .circleci/config.yml | 2 +- commands.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a98499d584..50b41fb10e 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -426,7 +426,7 @@ jobs: "PATH=$PATH:/home/zap/.local/bin && pip install wait-for-it && wait-for-it --service http://web:8080 \ - --timeout 60 \ + --timeout 120 \ -- echo \"Django is ready\"" - run-owasp-scan: environment: circle diff --git a/commands.sh b/commands.sh index e3571bea7c..b9664514ea 100644 --- a/commands.sh +++ b/commands.sh @@ -156,7 +156,7 @@ tdrs-run-backend-owasp() { "PATH=$PATH:/home/zap/.local/bin && pip install wait-for-it && wait-for-it --service http://web:8080 \ - --timeout 60 \ + --timeout 120 \ -- echo \"Django is ready\"" cd .. ./scripts/zap-scanner.sh backend circle From 4dac74dc67f4a6d4e6c8866efce2d0faa79bb509 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 25 Aug 2022 20:26:50 -0400 Subject: [PATCH 097/113] increase timeout for OWASP --- .circleci/config.yml | 2 +- commands.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50b41fb10e..9d6c37a51e 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -426,7 +426,7 @@ jobs: "PATH=$PATH:/home/zap/.local/bin && pip install wait-for-it && wait-for-it --service http://web:8080 \ - --timeout 120 \ + --timeout 360 \ -- echo \"Django is ready\"" - run-owasp-scan: environment: circle diff --git a/commands.sh b/commands.sh index b9664514ea..e3571bea7c 100644 --- a/commands.sh +++ b/commands.sh @@ -156,7 +156,7 @@ tdrs-run-backend-owasp() { "PATH=$PATH:/home/zap/.local/bin && pip install wait-for-it && wait-for-it --service http://web:8080 \ - --timeout 120 \ + --timeout 60 \ -- echo \"Django is ready\"" cd .. ./scripts/zap-scanner.sh backend circle From 4681a0ecfc4a550829f61fc0f77fd53d0642a15b Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 25 Aug 2022 20:52:00 -0400 Subject: [PATCH 098/113] Added env var for local Redis --- tdrs-backend/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 7229530d4d..be77570c7a 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -71,7 +71,7 @@ services: - ACFTITAN_KEY - ACFTITAN_USERNAME - REDIS_URI - - REDIS_SERVER_LOCAL + - REDIS_SERVER_LOCAL=True - ACFTITAN_SFTP_PYTEST volumes: - .:/tdpapp From f88eeeec2ea5cfe6fede538bec2ea577b1176442 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 25 Aug 2022 21:13:51 -0400 Subject: [PATCH 099/113] Added env var for owasp scan to use redis --- .circleci/config.yml | 2 +- tdrs-backend/gunicorn_start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d6c37a51e..a98499d584 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -426,7 +426,7 @@ jobs: "PATH=$PATH:/home/zap/.local/bin && pip install wait-for-it && wait-for-it --service http://web:8080 \ - --timeout 360 \ + --timeout 60 \ -- echo \"Django is ready\"" - run-owasp-scan: environment: circle diff --git a/tdrs-backend/gunicorn_start.sh b/tdrs-backend/gunicorn_start.sh index 894dc9f54a..2f5d76c93e 100755 --- a/tdrs-backend/gunicorn_start.sh +++ b/tdrs-backend/gunicorn_start.sh @@ -4,7 +4,7 @@ set -e echo "REDIS_SERVER" echo $REDIS_SERVER_LOCAL -if [[ "$REDIS_SERVER_LOCAL" = "TRUE" ]]; then +if [[ "$REDIS_SERVER_LOCAL" = "TRUE" || "$CIRCLE_JOB" = "backend-owasp-scan" ]]; then echo "Run redis server on docker" else echo "Run redis server locally" From 887a210260eaa4d7e619f01eee41eea1f4dbe2cf Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 25 Aug 2022 21:29:47 -0400 Subject: [PATCH 100/113] Changed redis_server_local to TRUE --- tdrs-backend/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index be77570c7a..666a2e346b 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -71,7 +71,7 @@ services: - ACFTITAN_KEY - ACFTITAN_USERNAME - REDIS_URI - - REDIS_SERVER_LOCAL=True + - REDIS_SERVER_LOCAL=TRUE - ACFTITAN_SFTP_PYTEST volumes: - .:/tdpapp From 271dc53f81d614b0ac6790275c88d6bd6c73d327 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Fri, 26 Aug 2022 12:11:46 -0400 Subject: [PATCH 101/113] corrected the directory config --- tdrs-backend/tdpservice/scheduling/sftp_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/sftp_task.py b/tdrs-backend/tdpservice/scheduling/sftp_task.py index b65d408cee..0e41ecce6b 100644 --- a/tdrs-backend/tdpservice/scheduling/sftp_task.py +++ b/tdrs-backend/tdpservice/scheduling/sftp_task.py @@ -85,9 +85,9 @@ def create_dir(directory_name, sftp_server): sftp = transport.open_sftp() # Create remote directory + create_dir(settings.ACFTITAN_DIRECTORY, sftp_server=sftp) create_dir(upper_directory_name, sftp_server=sftp) create_dir(lower_directory_name, sftp_server=sftp) - create_dir(settings.ACFTITAN_DIRECTORY, sftp_server=sftp) # Put the file in SFTP server sftp.put(destination, destination) From c1ea90231077a9cf5333c3e455d78f815dd2d89d Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Fri, 26 Aug 2022 14:33:21 -0400 Subject: [PATCH 102/113] Update tdrs-backend/tdpservice/scheduling/test/test_file_upload.py Co-authored-by: Alex P. <63075587+ADPennington@users.noreply.github.com> --- tdrs-backend/tdpservice/scheduling/test/test_file_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index 55ba6af2ae..f994f9dabb 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -22,7 +22,7 @@ def stt_instance(region): name="first", region=region, code="AR", - stt_codee=1234, + stt_code=1234, ) return stt From 82b508e1e69d695963536699abacbbdb0bda8725 Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Tue, 6 Sep 2022 10:22:45 -0400 Subject: [PATCH 103/113] Update sftp_upload task to use filename Updated the upload task to use filename instead of create_filename. The filename() method returns the filename based on STT code --- tdrs-backend/tdpservice/scheduling/sftp_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/sftp_task.py b/tdrs-backend/tdpservice/scheduling/sftp_task.py index 0e41ecce6b..dc5b1a29fd 100644 --- a/tdrs-backend/tdpservice/scheduling/sftp_task.py +++ b/tdrs-backend/tdpservice/scheduling/sftp_task.py @@ -33,7 +33,7 @@ def upload(data_file_pk, file_transfer_record = LegacyFileTransfer( data_file=data_file, uploaded_by=data_file.user, - file_name=data_file.create_filename(), + file_name=data_file.filename(), ) def write_key_to_file(private_key): From 728a71d21a992cdfa9f8908d4543d8ad20801aab Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 6 Sep 2022 11:10:47 -0400 Subject: [PATCH 104/113] changed stt.filename to return Tribal filename and added comment on TODO refactor --- tdrs-backend/tdpservice/data_files/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index d8738061ea..ad9a358fc9 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -149,10 +149,13 @@ class Meta: @property def filename(self): """Return the correct filename for this data file.""" - return self.stt.filenames.get(self.section, self.create_filename()) + return self.stt.filenames.get( + self.section if self.stt.type == 'state' else 'Tribal ' + self.section, + self.create_filename()) def create_filename(self, prefix='ADS.E2J'): """Return a valid file name for sftp transfer.""" + """TODO: This method has to be changed when the """ # STT_TYPES = ["state", "territory", "tribe"] SECTION = [i.value for i in list(self.Section)] From 29259957e56e926039f7123604f27414a8f836b1 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 7 Sep 2022 10:09:45 -0400 Subject: [PATCH 105/113] changed method to use property --- tdrs-backend/tdpservice/scheduling/sftp_task.py | 4 ++-- .../tdpservice/scheduling/test/test_file_upload.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/scheduling/sftp_task.py b/tdrs-backend/tdpservice/scheduling/sftp_task.py index dc5b1a29fd..2e85cf971e 100644 --- a/tdrs-backend/tdpservice/scheduling/sftp_task.py +++ b/tdrs-backend/tdpservice/scheduling/sftp_task.py @@ -33,7 +33,7 @@ def upload(data_file_pk, file_transfer_record = LegacyFileTransfer( data_file=data_file, uploaded_by=data_file.user, - file_name=data_file.filename(), + file_name=data_file.filename, ) def write_key_to_file(private_key): @@ -53,7 +53,7 @@ def create_dir(directory_name, sftp_server): try: # Create directory names for ACF titan - destination = str(data_file.create_filename()) + destination = str(data_file.filename) today_date = datetime.datetime.today() upper_directory_name = today_date.strftime('%Y%m%d') lower_directory_name = today_date.strftime(str(data_file.year) + '-' + str(data_file.quarter)) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index f994f9dabb..a82329804e 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -23,14 +23,19 @@ def stt_instance(region): region=region, code="AR", stt_code=1234, + filenames={ + 'Aggregate Data': 'ADS.E2J.NDM3.TS22', + 'Active Case Data': 'test', + 'Closed Case Data': 'ADS.E2J.NDM2.TS22'} ) return stt @pytest.fixture -def data_file_instance(): +def data_file_instance(stt_instance): """Prepare data file fixture instance for testing datafile.""" return DataFileFactory.create( created_at=datetime.now(), + stt = stt_instance ) @@ -83,4 +88,4 @@ def test_new_data_file(sftpserver, data_file_instance, sftp_connection_values, s '-' + str(data_file_instance.quarter)) assert sftpclient.listdir(upper_directory_name+'/'+lower_directory_name)[0] == \ - data_file_instance.create_filename() + data_file_instance.filename From d2513701bac3e8a208eaa5729c402e8f909fc9d8 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Wed, 7 Sep 2022 10:18:50 -0400 Subject: [PATCH 106/113] linting --- tdrs-backend/tdpservice/scheduling/test/test_file_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py index a82329804e..03757f63be 100644 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py @@ -35,7 +35,7 @@ def data_file_instance(stt_instance): """Prepare data file fixture instance for testing datafile.""" return DataFileFactory.create( created_at=datetime.now(), - stt = stt_instance + stt=stt_instance ) From 642d573c017abf42068b4516a7bc8e1640b164c3 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Thu, 8 Sep 2022 10:38:56 -0400 Subject: [PATCH 107/113] linting + changed test file --- tdrs-backend/tdpservice/data_files/models.py | 9 ++++++--- tdrs-backend/tdpservice/data_files/test/test_models.py | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index ad9a358fc9..1526b274e3 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -149,9 +149,12 @@ class Meta: @property def filename(self): """Return the correct filename for this data file.""" - return self.stt.filenames.get( - self.section if self.stt.type == 'state' else 'Tribal ' + self.section, - self.create_filename()) + # TODO: This is interim logic, it has to be changed when all sections are available to requester + if self.stt.type == 'tribe': + return self.stt.filenames.get('Tribal ' if 'Tribal' not in self.section else '' + self.section, + self.create_filename()) + else: + return self.stt.filenames.get(self.section, self.create_filename()) def create_filename(self, prefix='ADS.E2J'): """Return a valid file name for sftp transfer.""" diff --git a/tdrs-backend/tdpservice/data_files/test/test_models.py b/tdrs-backend/tdpservice/data_files/test/test_models.py index 8dd2de055f..b530cdd0fd 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_models.py +++ b/tdrs-backend/tdpservice/data_files/test/test_models.py @@ -81,4 +81,7 @@ def test_data_files_filename_is_expected(user): "user": user, "stt": stt }) - assert new_data_file.filename == stt.filenames[section] + if stt.type == 'tribe': + assert new_data_file.filename == stt.filenames['Tribal ' if 'Tribal' not in section else '' + section] + else: + assert new_data_file.filename == stt.filenames[section] From 6c8f2e613a217d261da49b3d37a8f7e246d7b204 Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Mon, 12 Sep 2022 11:01:38 -0400 Subject: [PATCH 108/113] Update models.py --- tdrs-backend/tdpservice/data_files/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index 1526b274e3..1c7e08f1a0 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -150,7 +150,7 @@ class Meta: def filename(self): """Return the correct filename for this data file.""" # TODO: This is interim logic, it has to be changed when all sections are available to requester - if self.stt.type == 'tribe': + if self.stt.type == 'tribe' or self.stt.type == 'TRIBE': return self.stt.filenames.get('Tribal ' if 'Tribal' not in self.section else '' + self.section, self.create_filename()) else: From 6cc737c9084b0d5cd3bcfe25814147e46cb0407a Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Mon, 12 Sep 2022 21:06:59 -0400 Subject: [PATCH 109/113] corrected Tribe logic --- tdrs-backend/tdpservice/data_files/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index 1526b274e3..728f204e2b 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -150,7 +150,7 @@ class Meta: def filename(self): """Return the correct filename for this data file.""" # TODO: This is interim logic, it has to be changed when all sections are available to requester - if self.stt.type == 'tribe': + if str(self.stt.type).lower() == 'tribe': return self.stt.filenames.get('Tribal ' if 'Tribal' not in self.section else '' + self.section, self.create_filename()) else: From ce849d70429bbb989038205ed8475c64d1a5cae6 Mon Sep 17 00:00:00 2001 From: Cameron Smart Date: Tue, 13 Sep 2022 12:41:07 -0700 Subject: [PATCH 110/113] force deployment when label is set to QASP REVIEW --- .github/workflows/deploy-on-label.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 0e42633b14..96c8fec142 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -58,6 +58,11 @@ jobs: STATE=${{ fromJson(steps.get-pr-checks.outputs.data).state }} echo "::set-output name=STATE::$STATE" echo "Current PR state: $STATE" + - name: Set success on QASP label since it is broken + if: ${{ github.event.label.name == 'QASP Review' }} + run: | + STATE="success" + echo "::set-output name=STATE::success" - name: Circle CI Deployment Trigger id: curl-circle-ci if: steps.get-pr-state.outputs.STATE == 'success' From 3bb65174a88705462c1ee82f62554531528c46e4 Mon Sep 17 00:00:00 2001 From: raftmsohani Date: Tue, 13 Sep 2022 17:41:27 -0400 Subject: [PATCH 111/113] changed the logic --- tdrs-backend/tdpservice/data_files/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index 1526b274e3..3ac1ed1ff1 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -150,8 +150,8 @@ class Meta: def filename(self): """Return the correct filename for this data file.""" # TODO: This is interim logic, it has to be changed when all sections are available to requester - if self.stt.type == 'tribe': - return self.stt.filenames.get('Tribal ' if 'Tribal' not in self.section else '' + self.section, + if str(self.stt.type).lower() == 'tribe': + return self.stt.filenames.get(('Tribal ' if 'Tribal' not in self.section else '') + self.section, self.create_filename()) else: return self.stt.filenames.get(self.section, self.create_filename()) From 1e0e4fb1efa687f8ad235ee8b8159df0dbb172c2 Mon Sep 17 00:00:00 2001 From: Cameron Smart Date: Tue, 13 Sep 2022 14:51:26 -0700 Subject: [PATCH 112/113] removed label force --- .github/workflows/deploy-on-label.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/deploy-on-label.yml b/.github/workflows/deploy-on-label.yml index 96c8fec142..0e42633b14 100644 --- a/.github/workflows/deploy-on-label.yml +++ b/.github/workflows/deploy-on-label.yml @@ -58,11 +58,6 @@ jobs: STATE=${{ fromJson(steps.get-pr-checks.outputs.data).state }} echo "::set-output name=STATE::$STATE" echo "Current PR state: $STATE" - - name: Set success on QASP label since it is broken - if: ${{ github.event.label.name == 'QASP Review' }} - run: | - STATE="success" - echo "::set-output name=STATE::success" - name: Circle CI Deployment Trigger id: curl-circle-ci if: steps.get-pr-state.outputs.STATE == 'success' From af5f23b85b6f53ddab3d25be4685209c33c85cfe Mon Sep 17 00:00:00 2001 From: raftmsohani <97037188+raftmsohani@users.noreply.github.com> Date: Wed, 14 Sep 2022 10:16:04 -0400 Subject: [PATCH 113/113] Updated the comment --- tdrs-backend/tdpservice/data_files/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index 3ac1ed1ff1..4a7b52ece1 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -158,7 +158,7 @@ def filename(self): def create_filename(self, prefix='ADS.E2J'): """Return a valid file name for sftp transfer.""" - """TODO: This method has to be changed when the """ + """TODO: This method has to be removed""" # STT_TYPES = ["state", "territory", "tribe"] SECTION = [i.value for i in list(self.Section)]