Merge pull request #360 from NASA-PDS/283-weekly-roundup

283 weekly roundup email notification

README.md

@@ -18,6 +18,8 @@ Please visit the documentation at: https://nasa-pds.github.io/doi-service/
 
 ## Developers
 
+[JPL Internal Wiki](https://wiki.jpl.nasa.gov/display/PDSEN/DOI+Service)
+
 Get the code and work on a branch:
 
     git clone ...

scripts/email_weekly_roundup.py

@@ -0,0 +1,26 @@
+import os.path
+from pds_doi_service.core.actions.roundup import run as run_weekly_roundup
+from pds_doi_service.core.db.doi_database import DOIDataBase
+
+from pds_doi_service.core.util.config_parser import DOIConfigUtil
+from pds_doi_service.core.util.general_util import get_logger
+
+if __name__ == '__main__':
+    """
+    Send an email consisting of a summary of all DOIs updated in the previous week (i.e. between the previous Sunday
+    and the Monday before that, inclusive), with a JSON attachment for those DoiRecords.
+
+    Should be run in a crontab, preferably on Monday, for example:
+    0 0 * * MON . path/to/doi-service/venv/bin/python path/to/doi-service/scripts/email_weekly_roundup.py
+    """
+
+    logging = get_logger('email_weekly_roundup')
+    config = DOIConfigUtil.get_config()
+    db_filepath = os.path.abspath(config['OTHER']['db_file'])
+    sender_email_address = config['OTHER']['emailer_sender']
+    receiver_email_address = config['OTHER']['emailer_receivers']
+    db = DOIDataBase(db_filepath)
+
+    run_weekly_roundup(db, sender_email_address, receiver_email_address)
+
+    logging.info('Completed DOI weekly roundup email transmission')

src/pds_doi_service/api/controllers/authentication.py

@@ -1,16 +1,19 @@
-import time
 import logging
-from jose import JWTError, jwt
+import time
+
 from flask import current_app
-from werkzeug.exceptions import Unauthorized
+from jose import jwt
+from jose import JWTError
 from pds_doi_service.core.util.config_parser import DOIConfigUtil
+from werkzeug.exceptions import Unauthorized
 
 config = DOIConfigUtil().get_config()
 
-JWT_ISSUER = config.get('API_AUTHENTICATION', 'jwt_issuer')
-JSON_WEB_KEY_SET = config.get('API_AUTHENTICATION', 'json_web_key_set')
-JWT_LIFETIME_SECONDS = config.get('API_AUTHENTICATION', 'jwt_lifetime_seconds')
-JWT_ALGORITHM = config.get('API_AUTHENTICATION', 'jwt_algorithm')
+JWT_ISSUER = config.get("API_AUTHENTICATION", "jwt_issuer")
+JSON_WEB_KEY_SET = config.get("API_AUTHENTICATION", "json_web_key_set")
+JWT_LIFETIME_SECONDS = config.get("API_AUTHENTICATION", "jwt_lifetime_seconds")
+JWT_ALGORITHM = config.get("API_AUTHENTICATION", "jwt_algorithm")
+
 
 def decode_token(token):
     try:
@@ -20,10 +23,7 @@ def decode_token(token):
             JSON_WEB_KEY_SET,
             algorithms=[JWT_ALGORITHM],
             issuer=JWT_ISSUER,
-            options={
-                'verify_signature': True,
-                'verify_iss': True
-            }
+            options={"verify_signature": True, "verify_iss": True},
         )
     except JWTError as e:
         current_app.logger.error("authentication exception")

src/pds_doi_service/core/actions/check.py

@@ -12,6 +12,7 @@
 Contains the definition for the Check action of the Core PDS DOI Service.
 """
 import json
+import os
 from copy import deepcopy
 from datetime import date
 from datetime import datetime
@@ -54,9 +55,12 @@ def __init__(self, db_name=None):
         self._email = True
         self._attachment = True
 
-        self.email_header_template_file = resource_filename(__name__, "email_template_header.txt")
-
-        self.email_body_template_file = resource_filename(__name__, "email_template_body.txt")
+        self.email_header_template_file = resource_filename(
+            __name__, os.path.join("templates", "email_template_header.txt")
+        )
+        self.email_body_template_file = resource_filename(
+            __name__, os.path.join("templates", "email_template_body.txt")
+        )
 
         # Make sure templates are where we expect them to be
         if not exists(self.email_header_template_file) or not exists(self.email_body_template_file):

src/pds_doi_service/core/actions/roundup.py

@@ -0,0 +1,124 @@
+#
+#  Copyright 2022, by the California Institute of Technology.  ALL RIGHTS
+#  RESERVED. United States Government Sponsorship acknowledged. Any commercial
+#  use must be negotiated with the Office of Technology Transfer at the
+#  California Institute of Technology.
+#
+"""
+======
+roundup.py
+======
+
+Contains functions for sending email notifications for recently-updated DOIs.
+"""
+import json
+import logging
+import os
+from datetime import date
+from datetime import datetime
+from datetime import timedelta
+from email import encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from typing import Dict
+from typing import List
+
+import jinja2
+from pds_doi_service.core.db.doi_database import DOIDataBase
+from pds_doi_service.core.entities.doi import DoiRecord
+from pds_doi_service.core.util.emailer import Emailer as PDSEmailer
+from pkg_resources import resource_filename
+
+
+def get_start_of_local_week() -> datetime:
+    """Return the start of the local-timezones week as a tz-aware datetime"""
+    today = datetime.now().date()
+    start_of_today = datetime(today.year, today.month, today.day)
+    return start_of_today.astimezone() - timedelta(days=today.weekday())
+
+
+def fetch_dois_modified_between(begin: datetime, end: datetime, database: DOIDataBase) -> List[DoiRecord]:
+    doi_records = database.select_latest_records({})
+    return [r for r in doi_records if begin <= r.date_added < end or begin <= r.date_updated < end]
+
+
+def get_email_content_template(template_filename: str = "email_weekly_roundup.jinja2"):
+    template_filepath = resource_filename(__name__, os.path.join("templates", template_filename))
+    logging.info(f"Using template {template_filepath}")
+    with open(template_filepath, "r") as infile:
+        template = jinja2.Template(infile.read())
+    return template
+
+
+def prepare_doi_record_for_template(record: DoiRecord) -> Dict[str, str]:
+    """Map a DoiRecord to the set of information required for rendering it in the template"""
+    update_type = "submitted" if record.date_added == record.date_updated else "updated"
+    prepared_record = {
+        "datacite_id": record.doi,
+        "pds_id": record.identifier,
+        "update_type": update_type,
+        "last_modified": record.date_updated,
+        "status": record.status.title(),
+    }
+
+    return prepared_record
+
+
+def prepare_email_html_content(first_date: date, last_date: date, modified_doi_records: List[DoiRecord]) -> str:
+    template = get_email_content_template()
+    template_dict = {
+        "first_date": first_date,
+        "last_date": last_date,
+        "doi_records": [prepare_doi_record_for_template(r) for r in modified_doi_records],
+    }
+
+    full_content = template.render(template_dict)
+    logging.info(full_content)
+    return full_content
+
+
+def attach_json_data(filename: str, doi_records: List[DoiRecord], msg: MIMEMultipart) -> None:
+    data = json.dumps([r.to_json_dict() for r in doi_records])
+    part = MIMEText(data, "plain", "utf-8")
+    part.set_charset("utf-8")
+    encoders.encode_base64(part)
+    part.add_header("Content-Disposition", f"attachment; filename={filename}")
+    msg.attach(part)
+
+
+def prepare_email_message(
+    sender_email: str, receiver_email: str, first_date: date, last_date: date, modified_doi_records: List[DoiRecord]
+) -> MIMEMultipart:
+    email_subject = f"DOI WEEKLY ROUNDUP: {first_date} through {last_date}"
+
+    msg = MIMEMultipart()
+    msg["From"] = sender_email
+    msg["Subject"] = email_subject
+    msg["To"] = receiver_email
+
+    email_content = prepare_email_html_content(first_date, last_date, modified_doi_records)
+    msg.attach(MIMEText(email_content, "html"))
+    attachment_filename = f"updated_dois_{first_date.isoformat()}_{last_date.isoformat()}.json"
+    attach_json_data(attachment_filename, modified_doi_records, msg)
+
+    return msg
+
+
+def run(database: DOIDataBase, sender_email: str, receiver_email: str) -> None:
+    """
+    Send an email consisting of a summary of all DOIs updated in the previous week (i.e. between the previous Sunday
+    and the Monday before that, inclusive), with a JSON attachment for those DoiRecords.
+    """
+    target_week_begin = get_start_of_local_week() - timedelta(days=7)
+    target_week_end = target_week_begin + timedelta(days=7, microseconds=-1)
+    last_date_of_week = (target_week_end - timedelta(microseconds=1)).date()
+
+    modified_doi_records = fetch_dois_modified_between(target_week_begin, target_week_end, database)
+
+    msg = prepare_email_message(
+        sender_email, receiver_email, target_week_begin.date(), last_date_of_week, modified_doi_records
+    )
+
+    emailer = PDSEmailer()
+    emailer.send_message(msg)

src/pds_doi_service/core/actions/templates/email_weekly_roundup.jinja2

@@ -0,0 +1,13 @@
+<h2>DataCite DOI Weekly Roundup</h2>
+{% if doi_records %}
+    <p>The following DOIs were reserved, updated or released during the week {{ first_date.strftime('%m/%d') }} through {{ last_date.strftime('%m/%d') }}:</p>
+    <ul>
+        {% for doi in doi_records %}
+            <li>{{ doi.datacite_id }} {{ doi.pds_id }} ({{ doi.status }}) {{ doi.update_type }} {{ doi.last_modified.strftime('%Y-%m-%d') }}</li>
+        {% endfor %}
+    </ul>
+{% else %}
+    <p>No DOI records were reserved, updated or released  modified during the week {{ first_date.strftime('%m/%d') }} through {{ last_date.strftime('%m/%d') }}.</p>
+{% endif %}
+
+<p><a href="mailto:Jordan.Padams@jpl.nasa.gov?subject=doi-service DOI Weekly Roundup">Contact Us</a></p>

src/pds_doi_service/core/actions/test/check_test.py

@@ -16,6 +16,8 @@
 import pds_doi_service.core.outputs.datacite.datacite_web_client
 import pds_doi_service.core.outputs.osti.osti_web_client
 from pds_doi_service.core.actions import DOICoreActionCheck
+from pds_doi_service.core.actions.test.util.email import capture_email
+from pds_doi_service.core.actions.test.util.email import get_local_smtp_patched_config
 from pds_doi_service.core.db.doi_database import DOIDataBase
 from pds_doi_service.core.entities.doi import DoiRecord
 from pds_doi_service.core.entities.doi import DoiStatus
@@ -188,26 +190,7 @@ def test_check_for_pending_entries_w_no_change(self):
         self.assertEqual(pending_record["doi"], "10.17189/29348")
         self.assertEqual(pending_record["identifier"], "urn:nasa:pds:lab_shocked_feldspars::1.0")
 
-    def get_config_patch(self):
-        """
-        Return a modified default config that points to a local test smtp
-        server for use with the email test
-        """
-        parser = configparser.ConfigParser()
-
-        # default configuration
-        conf_default = "conf.ini.default"
-        conf_default_path = abspath(join(dirname(__file__), os.pardir, os.pardir, "util", conf_default))
-
-        parser.read(conf_default_path)
-        parser["OTHER"]["emailer_local_host"] = "localhost"
-        parser["OTHER"]["emailer_port"] = "1025"
-
-        parser = DOIConfigUtil._resolve_relative_path(parser)
-
-        return parser
-
-    @patch.object(pds_doi_service.core.util.config_parser.DOIConfigUtil, "get_config", get_config_patch)
+    @patch.object(pds_doi_service.core.util.config_parser.DOIConfigUtil, "get_config", get_local_smtp_patched_config)
     @patch.object(
         pds_doi_service.core.outputs.osti.osti_web_client.DOIOstiWebClient, "query_doi", webclient_query_patch_nominal
     )
@@ -221,42 +204,20 @@ def test_email_receipt(self):
         # Create a new check action so our patched config is pulled in
         action = DOICoreActionCheck(self.db_name)
 
-        with tempfile.TemporaryFile() as temp_file:
-            # Stand up a subprocess running a debug smtpd server
-            # By default, all this server is does is echo email payloads to
-            # standard out, so provide a temp file to capture it
-            debug_email_proc = subprocess.Popen(
-                ["python", "-u", "-m", "smtpd", "-n", "-c", "DebuggingServer", "localhost:1025"], stdout=temp_file
-            )
-
-            # Give the debug smtp server a chance to start listening
-            time.sleep(1)
-
-            try:
-                # Run the check action and have it send an email w/ attachment
-                action.run(email=True, attachment=True, submitter="email-test@email.com")
-
-                # Read the raw email contents (payload) from the subprocess
-                # into a string
-                temp_file.seek(0)
-                email_contents = temp_file.read()
-                message = message_from_bytes(email_contents).get_payload()
-            finally:
-                # Send the debug smtp server a ctrl+C and wait for it to stop
-                os.kill(debug_email_proc.pid, signal.SIGINT)
-                debug_email_proc.wait()
-
-        # Run some string searches on the email body to ensure what we expect
-        # made it in
-
-        # Email address provided to check action should be present
-        self.assertIn("email-test@email.com", message)
+        message = capture_email(lambda: action.run(email=True, attachment=True, submitter="email-test@email.com"))
+        html_part, attachment_part = message.get_payload()
+        attachment = attachment_part.get_payload()[0].get_payload()[0]
+
+        # Email address provided to check action should be present in recipients
+        recipients = message["To"].strip().split(", ")
+        self.assertIn("email-test@email.com", recipients)
 
         # Subject line should be present
-        self.assertIn("DOI Submission Status Report For Node", message)
+        self.assertTrue(message["Subject"].strip().startswith("DOI Submission Status Report For Node"))
 
         # Attachment should also be provided
-        self.assertIn("Content-Disposition: attachment; filename=doi_status_", message)
+        self.assertTrue(attachment["Content-Disposition"].startswith("attachment; filename=doi_status_"))
+        self.assertTrue(attachment["Content-Disposition"].endswith(".json"))
 
 
 if __name__ == "__main__":

src/pds_doi_service/core/actions/test/data/roundup/roundup_email_attachment.json

@@ -0,0 +1,30 @@
+[
+  {
+    "identifier": "urn:nasa:pds:product_11111::1.0",
+    "status": "Pending",
+    "date_added": "2022-09-09T17:44:30.165920+00:00",
+    "date_updated": "2022-09-09T17:44:30.165920+00:00",
+    "submitter": "img-submitter@jpl.nasa.gov",
+    "title": "Laboratory Shocked Feldspars Bundle",
+    "type": "Collection",
+    "subtype": "PDS4 Collection",
+    "node_id": "img",
+    "doi": "10.17189/11111",
+    "transaction_key": "./transaction_history/img/2020-06-15T18:42:45.653317",
+    "is_latest": true
+  },
+  {
+    "identifier": "urn:nasa:pds:product_22222::1.0",
+    "status": "Pending",
+    "date_added": "2022-08-14T17:44:30.165940+00:00",
+    "date_updated": "2022-09-09T17:44:30.165940+00:00",
+    "submitter": "img-submitter@jpl.nasa.gov",
+    "title": "Laboratory Shocked Feldspars Bundle",
+    "type": "Collection",
+    "subtype": "PDS4 Collection",
+    "node_id": "img",
+    "doi": "10.17189/22222",
+    "transaction_key": "./transaction_history/img/2020-06-15T18:42:45.653317",
+    "is_latest": true
+  }
+]

src/pds_doi_service/core/actions/test/data/roundup/roundup_email_body.jinja2

@@ -0,0 +1,13 @@
+<h2>DataCite DOI Weekly Roundup</h2>
+
+<p>The following DOIs were reserved, updated or released during the week {{ week_start.strftime('%m/%d') }} through {{ week_end.strftime('%m/%d') }}:</p>
+<ul>
+
+    <li>10.17189/11111 urn:nasa:pds:product_11111::1.0 (Pending) submitted {{ modifications_date.isoformat() }}</li>
+
+    <li>10.17189/22222 urn:nasa:pds:product_22222::1.0 (Pending) updated {{ modifications_date.isoformat() }}</li>
+
+</ul>
+
+
+<p><a href="mailto:Jordan.Padams@jpl.nasa.gov?subject=doi-service DOI Weekly Roundup">Contact Us</a></p>