Add Riprap as a microsevice (#113)

* Automated Tests

- Added automated tests and documentation.
- Return the exit code of the service that exited
- Previously always returned 0 even if the service failed to start.

* Bump version of plugin

* Added Riprap micro-service.

.vscode/settings.json

@@ -14,14 +14,16 @@
         "composer": "shellscript"
     },
     "cSpell.words": [
-        "MODESHAPE",
-        "POSTGRESQL",
-        "SIGTERM",
         "binarystorage",
         "catchable",
         "classpath",
+        "crond",
         "elif",
         "getenv",
-        "nativeplatform"
+        "getenv",
+        "MODESHAPE",
+        "nativeplatform",
+        "POSTGRESQL",
+        "SIGTERM"
     ]
 }

base/rootfs/etc/cont-init.d/00-container-environment-02-database-defaults.sh

@@ -20,8 +20,10 @@ case "${DB_DRIVER}" in
         DB_HOST=$(</var/run/s6/container_environment/DB_POSTGRESQL_HOST)
         DB_PORT=$(</var/run/s6/container_environment/DB_POSTGRESQL_PORT)
         ;;
+    sqlite)
+        ;;
     *)
-        echo "Only mysql or postgresql are supported values for DB_DRIVER." >&2
+        echo "Only MySQL / PostgreSQL / SQLite are supported values for DB_DRIVER." >&2
         exit 1
 esac
 

riprap/.dockerignore

@@ -0,0 +1 @@
+README.md

riprap/Dockerfile

@@ -0,0 +1,79 @@
+# syntax=docker/dockerfile:1.2.1
+ARG repository=local
+ARG tag=latest
+FROM --platform=$BUILDPLATFORM ${repository}/composer:${tag} AS composer
+
+RUN --mount=type=cache,id=riprap-composer,sharing=locked,target=/root/.composer/cache \
+    --mount=type=cache,id=riprap-downloads,sharing=locked,target=/opt/downloads \
+    COMMIT=7d7cae2d09dec20caa3c3f5752434af32401819e && \
+    git-clone-cached.sh \
+        --url https://github.com/mjordan/riprap.git \
+        --cache-dir "${DOWNLOAD_CACHE_DIRECTORY}" \
+        --commit "${COMMIT}" \
+        --worktree /var/www/riprap && \
+    composer install -d /var/www/riprap --no-dev
+
+FROM alpine:3.13.2 AS cache
+FROM ${repository}/nginx:${tag}
+
+EXPOSE 8000
+
+RUN --mount=type=cache,id=riprap-apk,sharing=locked,from=cache,target=/var/cache/apk \
+    apk add php7-pdo_sqlite && \
+    cleanup.sh
+
+# The driver is given explicitly as Rip Rap can be run on SQLite without
+# further configuration.
+ENV \
+    RIPRAP_APP_ENV=dev \
+    RIPRAP_APP_SECRET=f58c87e1d737c4422b45ba4310abede6 \
+    RIPRAP_CROND_ENABLE_SERVICE=true \
+    RIPRAP_CROND_LOG_LEVEL=8 \
+    RIPRAP_CROND_SCHEDULE="0	0	1	*	*" \
+    RIPRAP_DB_DRIVER=sqlite \
+    RIPRAP_DB_NAME=riprap \
+    RIPRAP_DB_PASSWORD=password \
+    RIPRAP_DB_USER=riprap \
+    RIPRAP_LOG_LEVEL=debug \
+    RIPRAP_MAILER_URL=null://localhost \
+    RIPRAP_TRUSTED_HOSTS="" \
+    RIPRAP_TRUSTED_PROXIES=""
+
+# Configuration specific to check fixity command:
+ENV \
+    RIPRAP_CONFIG_DIGEST_COMMAND=/usr/bin/sha1sum \
+    RIPRAP_CONFIG_DRUPAL_BASEURL=https://islandora.traefik.me \
+    RIPRAP_CONFIG_DRUPAL_CONTENT_TYPES="['islandora_object']" \
+    RIPRAP_CONFIG_DRUPAL_FILE_FIELDNAMES="['field_media_audio', 'field_media_document', 'field_edited_text', 'field_media_file', 'field_media_image', 'field_media_video_file']" \
+    RIPRAP_CONFIG_DRUPAL_MEDIA_AUTH="['admin', 'islandora']" \
+    RIPRAP_CONFIG_DRUPAL_MEDIA_TAGS="[]" \
+    RIPRAP_CONFIG_DRUPAL_PASSWORD=password \
+    RIPRAP_CONFIG_DRUPAL_USER=admin \
+    RIPRAP_CONFIG_EMAIL_FROM="" \
+    RIPRAP_CONFIG_EMAIL_TO="" \
+    RIPRAP_CONFIG_FAILURES_LOG_PATH=var/riprap_failed_events.log \
+    RIPRAP_CONFIG_FEDORAAPI_DIGEST_HEADER_LEADER_PATTERN="^.+=" \
+    RIPRAP_CONFIG_FEDORAAPI_METHOD=HEAD \
+    RIPRAP_CONFIG_FIXITY_ALGORITHM=sha1 \
+    RIPRAP_CONFIG_GEMINI_AUTH_HEADER="Bearer islandora" \
+    RIPRAP_CONFIG_GEMINI_ENDPOINT=http://gemini:8000 \
+    RIPRAP_CONFIG_JSONAPI_AUTHORIZATION_HEADERS="" \
+    RIPRAP_CONFIG_JSONAPI_PAGE_SIZE=50 \
+    RIPRAP_CONFIG_JSONAPI_PAGER_DATA_FILE_PATH=var/fetchresourcelist.from.drupal.pager.txt \
+    RIPRAP_CONFIG_MAX_RESOURCES=1000 \
+    RIPRAP_CONFIG_OUTPUT_CSV_PATH=var/riprap_events.csv \
+    RIPRAP_CONFIG_PLUGINS_FETCHDIGEST=PluginFetchDigestFromShell \
+    RIPRAP_CONFIG_PLUGINS_FETCHRESOURCELIST="['PluginFetchResourceListFromFile']" \
+    RIPRAP_CONFIG_PLUGINS_PERSIST=PluginPersistToDatabase \
+    RIPRAP_CONFIG_PLUGINS_POSTCHECK="['PluginPostCheckCopyFailures']" \
+    RIPRAP_CONFIG_RESOURCE_DIR_PATHS="" \
+    RIPRAP_CONFIG_RESOURCE_LIST_PATH="['resources/csv_file_list.csv']" \
+    RIPRAP_CONFIG_THIN=false \
+    RIPRAP_CONFIG_USE_FEDORA_URLS=true \
+    RIPRAP_CONFIG_VIEWS_PAGER_DATA_FILE_PATH=var/fetchresourcelist.from.drupal.pager.txt
+
+COPY --from=composer --chown=nginx:nginx /var/www /var/www
+
+COPY rootfs /
+
+WORKDIR /var/www/riprap

riprap/rootfs/etc/confd/conf.d/.env.toml

@@ -0,0 +1,7 @@
+[template]
+src = ".env.tmpl"
+dest = "/var/www/riprap/.env"
+uid = 100
+gid = 101
+mode = "0644"
+keys = [ "/" ]

riprap/rootfs/etc/confd/conf.d/cron_config.yml.toml

@@ -0,0 +1,7 @@
+[template]
+src = "cron_config.yml.tmpl"
+dest = "/var/www/riprap/cron_config.yaml"
+uid = 100
+gid = 101
+mode = "0644"
+keys = [ "/" ]

riprap/rootfs/etc/confd/conf.d/doctrine.yaml.toml

@@ -0,0 +1,7 @@
+[template]
+src = "doctrine.yaml.tmpl"
+dest = "/var/www/riprap/config/packages/doctrine.yaml"
+uid = 100
+gid = 101
+mode = "0644"
+keys = [ "/" ]

riprap/rootfs/etc/confd/conf.d/monolog.yaml.toml

@@ -0,0 +1,7 @@
+[template]
+src = "monolog.yaml.tmpl"
+dest = "/var/www/riprap/config/packages/dev/monolog.yaml"
+uid = 100
+gid = 101
+mode = "0644"
+keys = [ "/" ]

riprap/rootfs/etc/confd/templates/.env.tmpl

@@ -0,0 +1,35 @@
+# This file is a "template" of which env vars need to be defined for your application
+# Copy this file to .env file for development, create environment variables when deploying to production
+# https://symfony.com/doc/current/best_practices/configuration.html#infrastructure-related-configuration
+
+###> symfony/framework-bundle ###
+APP_ENV={{ getenv "RIPRAP_APP_ENV" }}
+APP_SECRET={{ getenv "RIPRAP_APP_SECRET" }}
+{{ if ne (getenv "RIPRAP_TRUSTED_PROXIES") "" }}
+TRUSTED_PROXIES={{ getenv "RIPRAP_TRUSTED_PROXIES" }}
+{{ end }}
+{{ if ne (getenv "RIPRAP_TRUSTED_HOSTS") "" }}
+TRUSTED_HOSTS={{ getenv "RIPRAP_TRUSTED_HOSTS" }}
+{{ end }}
+###< symfony/framework-bundle ###
+
+###> doctrine/doctrine-bundle ###
+# Format described at http://docs.doctrine-project.org/projects/doctrine-dbal/en/latest/reference/configuration.html#connecting-using-a-url
+# Configure your db driver and server_version in config/packages/doctrine.yaml
+{{ if eq (getenv "DB_DRIVER") "sqlite" }}
+DATABASE_URL=sqlite:///%kernel.project_dir%/var/data.db
+{{ end }}
+{{ if eq (getenv "DB_DRIVER") "mysql" }}
+DATABASE_URL=mysql://{{ getenv "RIPRAP_DB_USER" }}:{{ getenv "RIPRAP_DB_PASSWORD" }}@{{ getenv "DB_MYSQL_HOST" }}:{{ getenv "DB_MYSQL_PORT" }}/{{ getenv "RIPRAP_DB_NAME" }}
+{{ end }}
+{{ if eq (getenv "DB_DRIVER") "postgresql" }}
+DATABASE_URL=pgsql://{{ getenv "RIPRAP_DB_USER" }}:{{ getenv "RIPRAP_DB_PASSWORD" }}@{{ getenv "DB_POSTGRESQL_HOST" }}:{{ getenv "DB_POSTGRESQL_PORT" }}/{{ getenv "RIPRAP_DB_NAME" }}
+{{ end }}
+###< doctrine/doctrine-bundle ###
+
+###> symfony/swiftmailer-bundle ###
+# For Gmail as a transport, use: "gmail://username:password@localhost"
+# For a generic SMTP server, use: "smtp://localhost:25?encryption=&auth_mode="
+# Delivery is disabled by default via "null://localhost"
+MAILER_URL={{ getenv "RIPRAP_MAILER_URL" }}
+###< symfony/swiftmailer-bundle ###

riprap/rootfs/etc/confd/templates/cron_config.yml.tmpl

@@ -0,0 +1,40 @@
+# Riprap config file used by the crond service.
+#
+# Requires that the "Riprap resource list" View be enabled in
+# the Islandora instance. This View is bundled with the Islandora Riprap
+# module.
+#
+# This plugin is agnostic to which media have fixity event checks performed
+# on them. The filters in the "Riprap resource list" View determine that.
+# See the View's filter criteria GUI for examples.
+
+digest_command: '{{ getenv "RIPRAP_CONFIG_DIGEST_COMMAND" }}'
+drupal_baseurl: '{{ getenv "RIPRAP_CONFIG_DRUPAL_BASEURL" }}'
+drupal_content_types: {{ getenv "RIPRAP_CONFIG_DRUPAL_CONTENT_TYPES" }}
+drupal_file_fieldnames: {{ getenv "RIPRAP_CONFIG_DRUPAL_FILE_FIELDNAMES" }}
+drupal_media_auth: {{ getenv "RIPRAP_CONFIG_DRUPAL_MEDIA_AUTH" }}
+drupal_media_tags: {{ getenv "RIPRAP_CONFIG_DRUPAL_MEDIA_TAGS" }}
+drupal_password: '{{ getenv "RIPRAP_CONFIG_DRUPAL_PASSWORD" }}'
+drupal_user: '{{ getenv "RIPRAP_CONFIG_DRUPAL_USER" }}'
+email_from: '{{ getenv "RIPRAP_CONFIG_EMAIL_FROM" }}'
+email_to: '{{ getenv "RIPRAP_CONFIG_EMAIL_TO" }}'
+failures_log_path: '{{ getenv "RIPRAP_CONFIG_FAILURES_LOG_PATH" }}'
+fedoraapi_digest_header_leader_pattern: '{{ getenv "RIPRAP_CONFIG_FEDORAAPI_DIGEST_HEADER_LEADER_PATTERN" }}'
+fedoraapi_method: '{{ getenv "RIPRAP_CONFIG_FEDORAAPI_METHOD" }}'
+fixity_algorithm: '{{ getenv "RIPRAP_CONFIG_FIXITY_ALGORITHM" }}'
+gemini_auth_header: '{{ getenv "RIPRAP_CONFIG_GEMINI_AUTH_HEADER" }}'
+gemini_endpoint: '{{ getenv "RIPRAP_CONFIG_GEMINI_ENDPOINT" }}'
+jsonapi_authorization_headers: '{{ getenv "RIPRAP_CONFIG_JSONAPI_AUTHORIZATION_HEADERS" }}'
+jsonapi_pager_data_file_path: '{{ getenv "RIPRAP_CONFIG_JSONAPI_PAGER_DATA_FILE_PATH" }}'
+jsonapi_page_size: {{ getenv "RIPRAP_CONFIG_JSONAPI_PAGE_SIZE" }}
+max_resources: {{ getenv "RIPRAP_CONFIG_MAX_RESOURCES" }}
+output_csv_path: '{{ getenv "RIPRAP_CONFIG_OUTPUT_CSV_PATH" }}'
+plugins.fetchdigest: '{{ getv "/config/plugins.fetchdigest" (getenv "RIPRAP_CONFIG_PLUGINS_FETCHDIGEST") }}'
+plugins.fetchresourcelist: {{ getv "/config/plugins.fetchresourcelist" (getenv "RIPRAP_CONFIG_PLUGINS_FETCHRESOURCELIST") }}
+plugins.persist: '{{ getv "/config/plugins.persist" (getenv "RIPRAP_CONFIG_PLUGINS_PERSIST") }}'
+plugins.postcheck: {{ getv "/config/plugins.postcheck" (getenv "RIPRAP_CONFIG_PLUGINS_POSTCHECK") }}
+resource_dir_paths: {{ getenv "RIPRAP_CONFIG_RESOURCE_DIR_PATHS" }}
+resource_list_path: {{ getenv "RIPRAP_CONFIG_RESOURCE_LIST_PATH" }}
+thin: {{ getenv "RIPRAP_CONFIG_THIN" }}
+use_fedora_urls: {{ getenv "RIPRAP_CONFIG_USE_FEDORA_URLS" }}
+views_pager_data_file_path: '{{ getenv "RIPRAP_CONFIG_VIEWS_PAGER_DATA_FILE_PATH" }}'

riprap/rootfs/etc/confd/templates/doctrine.yaml.tmpl

@@ -0,0 +1,32 @@
+parameters:
+    # Adds a fallback DATABASE_URL if the env var is not set.
+    # This allows you to run cache:warmup even if your
+    # environment variables are not available yet.
+    # You should not need to change this value.
+    env(DATABASE_URL): ''
+
+doctrine:
+    dbal:
+{{ if eq (getenv "DB_DRIVER") "mysql" }}
+        driver: 'pdo_mysql'
+        charset: utf8mb4
+        default_table_options:
+            charset: utf8mb4
+            collate: utf8mb4_unicode_ci
+{{ end }}
+{{ if eq (getenv "DB_DRIVER") "postgresql" }}
+        driver: 'pdo_pgsql'
+        charset: utf8
+{{ end }}
+        url: '%env(resolve:DATABASE_URL)%'
+    orm:
+        auto_generate_proxy_classes: '%kernel.debug%'
+        naming_strategy: doctrine.orm.naming_strategy.underscore_number_aware
+        auto_mapping: true
+        mappings:
+            App:
+                is_bundle: false
+                type: annotation
+                dir: '%kernel.project_dir%/src/Entity'
+                prefix: 'App\Entity'
+                alias: App

riprap/rootfs/etc/confd/templates/monolog.yaml.tmpl

@@ -0,0 +1,20 @@
+monolog:
+    handlers:
+        main:
+            type: stream
+            # path: "%kernel.logs_dir%/%kernel.environment%.log"
+            path: "php://stderr"
+            level: {{ getenv "RIPRAP_LOG_LEVEL" }}
+            channels: ["!event"]
+        # uncomment to get logging in your browser
+        # you may have to allow bigger header sizes in your Web server configuration
+        #firephp:
+        #    type: firephp
+        #    level: info
+        #chromephp:
+        #    type: chromephp
+        #    level: info
+        console:
+            type: console
+            process_psr_3_messages: false
+            channels: ["!event", "!doctrine", "!console"]

riprap/rootfs/etc/cont-init.d/03-riprap-setup.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/with-contenv bash
+set -e
+
+function mysql_create_database {
+    cat <<- EOF | create-database.sh
+-- Create database in mariadb or mysql. 
+CREATE DATABASE IF NOT EXISTS ${RIPRAP_DB_NAME} CHARACTER SET utf8 COLLATE utf8_general_ci;
+
+-- Create user and grant rights.
+CREATE USER IF NOT EXISTS '${RIPRAP_DB_USER}'@'%' IDENTIFIED BY '${RIPRAP_DB_PASSWORD}';
+GRANT ALL PRIVILEGES ON ${RIPRAP_DB_NAME}.* to '${RIPRAP_DB_USER}'@'%';
+FLUSH PRIVILEGES;
+
+-- Update user password if changed.
+SET PASSWORD FOR ${RIPRAP_DB_USER}@'%' = PASSWORD('${RIPRAP_DB_PASSWORD}')
+EOF
+}
+
+function postgresql_create_database {
+    cat <<- EOF | create-database.sh
+BEGIN;
+
+DO \$\$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${RIPRAP_DB_USER}') THEN
+        CREATE ROLE ${RIPRAP_DB_USER};
+    END IF;
+END
+\$\$;
+
+ALTER ROLE ${RIPRAP_DB_USER} WITH LOGIN;
+ALTER USER ${RIPRAP_DB_USER} PASSWORD '${RIPRAP_DB_PASSWORD}';
+
+ALTER DATABASE ${RIPRAP_DB_NAME} OWNER TO ${RIPRAP_DB_USER};
+GRANT ALL PRIVILEGES ON DATABASE ${RIPRAP_DB_NAME} TO ${RIPRAP_DB_USER};
+
+COMMIT;
+EOF
+}
+
+function create_database {
+    case "${DB_DRIVER}" in
+        sqlite)
+            # Running migrations will create the database.
+            ;;
+        mysql)
+            mysql_create_database
+            ;;
+        postgresql)
+            postgresql_create_database
+            ;;
+        *)
+            echo "Only SQLite/MySQL/PostgresSQL databases are supported for now." >&2
+            exit 1
+    esac
+}
+
+function setup_cron {
+    if [[ "${RIPRAP_CROND_ENABLE_SERVICE}" == "true" ]]; then
+        cat <<EOF | crontab -u nginx -
+# min	hour	day	month	weekday	command
+${RIPRAP_CROND_SCHEDULE}	check-fixity.sh --settings=/var/www/riprap/cron_config.yml
+EOF
+    fi
+}
+
+function run_migrations {
+    s6-setuidgid nginx php bin/console --no-interaction make:migration
+    local num_migrations=$(find src/Migrations -type f -name "*.php" | wc -l)
+    if [[ ${num_migrations} -gt 0 ]]; then
+        s6-setuidgid nginx php bin/console --no-interaction doctrine:migrations:migrate
+    fi
+}
+
+function main {
+    create_database
+    run_migrations
+    setup_cron
+}
+main

riprap/rootfs/etc/nginx/http.d/default.conf

@@ -0,0 +1,29 @@
+# From: https://www.nginx.com/resources/wiki/start/topics/recipes/drupal/
+server {
+    listen 8000;
+    root /var/www/riprap/public;
+
+    location / {
+        # try to serve file directly, fallback to index.php
+        try_files $uri /index.php$is_args$args;
+    }
+
+    location ~ ^/index\.php(/|$) {
+        fastcgi_pass unix:/var/run/php-fpm7/php-fpm7.sock;
+        fastcgi_split_path_info ^(.+\.php)(/.*)$;
+        include fastcgi_params;
+
+        fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name;
+        fastcgi_param DOCUMENT_ROOT $realpath_root;
+        # Prevents URIs that include the front controller. This will 404:
+        # http://domain.tld/index.php/some-path
+        # Remove the internal directive to allow URIs like this
+        internal;
+    }
+
+    # return 404 for all other php files not matching the front controller
+    # this prevents access to other php files you don't want to be accessible.
+    location ~ \.php$ {
+        return 404;
+    }
+}