Merge pull request #95 from Islandora-Devops/post-sprint-4

Sandbox, Import/Export tool, and Syn toggling

alpaca/Dockerfile

@@ -3,7 +3,7 @@ FROM local/karaf:latest
 
 # Install common features and repos
 RUN bin/start && \
-    ALPACA_VERSION=1.0.3 && \
+    ALPACA_VERSION=1.0.4 && \
     ACTIVEMQ_VERSION=5.15.0 && \
     CAMEL_VERSION=2.20.4 && \
     bin/client -r 10 -d 5  "feature:repo-add mvn:ca.islandora.alpaca/islandora-karaf/${ALPACA_VERSION}/xml/features" && \
@@ -29,7 +29,6 @@ RUN bin/start && \
 
 # Triple indexing
 RUN bin/start && \
-    bin/client -r 10 -d 5  "feature:install fcrepo-indexing-triplestore" && \
     bin/client -r 10 -d 5  "feature:install islandora-indexing-triplestore" && \
     bin/stop && \
     rm -rf instances/*
@@ -54,11 +53,11 @@ ENV \
     ALPACA_HOUDINI_REDELIVERIES=10 \
     ALPACA_HOUDINI_SERVICE=http://houdini:8000/convert \
     ALPACA_HTTP_TOKEN=islandora \
-    ALPACA_INDEXING_GEMINI_URL=http://gemini:8000 \
     ALPACA_INDEXING_MILLINER_URL=http://milliner:8000 \
     ALPACA_INDEXING_REDELIVERIES=10 \
     ALPACA_INDEXING_STREAM_FILE_DELETE=broker:queue:islandora-indexing-fcrepo-file-delete \
     ALPACA_INDEXING_STREAM_FILE_INDEX=broker:queue:islandora-indexing-fcrepo-file \
+    ALPACA_INDEXING_STREAM_FILE_EXTERNAL=broker:queue:islandora-indexing-fcrepo-file-external \
     ALPACA_INDEXING_STREAM_INPUT=broker:topic:fedora \
     ALPACA_INDEXING_STREAM_MEDIA_INDEX=broker:queue:islandora-indexing-fcrepo-media \
     ALPACA_INDEXING_STREAM_NODE_DELETE=broker:queue:islandora-indexing-fcrepo-delete \

alpaca/rootfs/etc/confd/templates/ca.islandora.alpaca.indexing.fcrepo.cfg.tmpl

@@ -1,7 +1,7 @@
 error.maxRedeliveries={{ getv "/indexing/redeliveries" (getenv "ALPACA_INDEXING_REDELIVERIES") }}
+file.external.stream={{ getv "/indexing/stream/file/external" (getenv "ALPACA_INDEXING_STREAM_FILE_EXTERNAL") }}
 file.delete.stream={{ getv "/indexing/stream/file/delete" (getenv "ALPACA_INDEXING_STREAM_FILE_DELETE") }}
 file.stream={{ getv "/indexing/stream/file/index" (getenv "ALPACA_INDEXING_STREAM_FILE_INDEX") }}
-gemini.baseUrl={{ getv "/indexing/gemini/url" (getenv "ALPACA_INDEXING_GEMINI_URL") }}
 media.stream={{ getv "/indexing/stream/media/index" (getenv "ALPACA_INDEXING_STREAM_MEDIA_INDEX") }}
 milliner.baseUrl={{ getv "/indexing/milliner/url" (getenv "ALPACA_INDEXING_MILLINER_URL") }}
 node.delete.stream={{ getv "/indexing/stream/node/delete" (getenv "ALPACA_INDEXING_STREAM_NODE_DELETE") }}

base/rootfs/etc/cont-init.d/00-container-environment-00-init.sh

@@ -80,4 +80,4 @@ with-contenv confd -prefix '/' -onetime -sync-only -confdir /tmp/confd -log-leve
 execlineb -P /tmp/confd/out/import.sh 
 
 # Remove temporary files.
-rm -fr /tmp/confd
+rm -fr /tmp/confd

base/rootfs/usr/local/bin/execute-sql-file.sh

@@ -162,7 +162,7 @@ function postgresql_execute_sql_file {
 
 function execute_sql_file {
     case "${DRIVER}" in
-        mysql|pdo_mysql)
+        mysql|pdo_mysql|mariadb)
             mysql_execute_sql_file
             ;;
         pgsql|postgresql|pdo_pgsql)

base/rootfs/usr/local/bin/wait-for-database.sh

@@ -123,7 +123,7 @@ function postgresql_validate_credentials {
 function validate_credentials {
     echo "Validating Database credentials"
     case "${DRIVER}" in
-        mysql|pdo_mysql)
+        mysql|pdo_mysql|mariadb)
             mysql_validate_credentials
             ;;
         pgsql|postgresql|pdo_pgsql)

crayfish/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1.2.1
 FROM local/nginx:latest
 
-ARG COMMIT=85a8206a9ed1db302fdeb123f9d5391ef8aae001
+ARG COMMIT=ae9a35c88d2bdfd0343122a8b5b373d6368cc623
 
 RUN --mount=id=downloads,type=cache,target=/opt/downloads \
     DOWNLOAD_CACHE_DIRECTORY="/opt/downloads" && \

demo/Dockerfile

@@ -1,60 +1,59 @@
 # syntax=docker/dockerfile:1.2.1
 FROM local/nginx:latest as composer
 
+ARG COMMIT=6e9976b156e109cd87c5398afdd9786a16b0773a
+
 # Islandora based Drupal install.
 RUN --mount=type=cache,target=/root/.composer/cache \
     --mount=id=downloads,type=cache,target=/opt/downloads \
     DOWNLOAD_CACHE_DIRECTORY="/opt/downloads" && \
-    composer create-project drupal/recommended-project:^8.9 \
-        --prefer-dist \
-        --no-interaction \
-        --stability stable \
-        --no-dev \
-        --no-install \
-        /var/www/drupal \
-    && \
+    # Get the Drupal codebase
+    git-clone-cached.sh \
+        --url https://github.com/dannylamb/islandora-sandbox.git \
+        --cache-dir "${DOWNLOAD_CACHE_DIRECTORY}" \
+        --commit "${COMMIT}" \
+        --worktree /var/www/drupal && \
     cd /var/www/drupal && \
-    composer require --update-no-dev -- \
-        drupal/admin_toolbar:^2.0 \
-        drupal/content_browser:^1.0@alpha \
-        drupal/devel:^2.0 \
-        drupal/facets:^1.3 \
-        drupal/matomo:^1.7 \
-        drupal/pdf:1.x-dev \
-        drupal/rdfui:^1.0-beta1 \
-        drupal/rest_oai_pmh:^1.0 \
-        drupal/restui:^1.16 \
-        drupal/search_api_solr:^3.8 \
-        drupal/transliterate_filenames:^1.3 \
-        drush/drush:^10.3 \
-        islandora-rdm/islandora_fits:dev-8.x-1.x \
-        islandora/carapace:dev-8.x-3.x \
-        islandora/islandora_defaults:dev-8.x-1.x \
-        zaporylie/composer-drupal-optimizations:^1.1 \
-    && \
+    composer install && \
+    # Set up libraries
     mkdir -p /var/www/drupal/web/libraries && \
-    MASONRY_VERSION="3.3.2" && \
-    MASONRY_FILE="v${MASONRY_VERSION}.zip" && \
-    MASONRY_URL="https://github.com/desandro/masonry/archive/${MASONRY_FILE}" && \
-    MASONRY_SHA256="85dc8b2cdf789693a14022dadc3e573a87d625da5da1cf83e1ef1e5af22af496" && \
-    download.sh --url "${MASONRY_URL}" --sha256 "${MASONRY_SHA256}" "${DOWNLOAD_CACHE_DIRECTORY}" && \
-    unzip "${DOWNLOAD_CACHE_DIRECTORY}/${MASONRY_FILE}" -d /var/www/drupal/web/libraries && \
-    mv  /var/www/drupal/web/libraries/masonry-${MASONRY_VERSION} /var/www/drupal/web/libraries/masonry && \
+    # PDF.js
     PDFJS_VERSION="2.0.943" && \
     PDFJS_FILE="pdfjs-${PDFJS_VERSION}-dist.zip" && \
     PDFJS_URL="https://github.com/mozilla/pdf.js/releases/download/v${PDFJS_VERSION}/${PDFJS_FILE}" && \
     PDFJS_SHA256="381683b05f494fe28e11185f98a238b34a663b1b41b5432aa769ca493e5cd087" && \
     download.sh --url "${PDFJS_URL}" --sha256 "${PDFJS_SHA256}" "${DOWNLOAD_CACHE_DIRECTORY}" && \
     mkdir -p /var/www/drupal/web/libraries/pdf.js && \
     unzip "${DOWNLOAD_CACHE_DIRECTORY}/${PDFJS_FILE}" -d /var/www/drupal/web/libraries/pdf.js && \
-    OPENSEADRAGON_VERSION="2.4.2" && \
-    OPENSEADRAGON_FILE="openseadragon-bin-${OPENSEADRAGON_VERSION}.zip" && \
-    OPENSEADRAGON_URL="https://github.com/openseadragon/openseadragon/releases/download/v${OPENSEADRAGON_VERSION}/${OPENSEADRAGON_FILE}" && \
-    OPENSEADRAGON_SHA256="8e32d808b80206d5f5aadcfc51ffecf2ecc89e5fb6876c51f409940cf4b1087e" && \
-    download.sh --url "${OPENSEADRAGON_URL}" --sha256 "${OPENSEADRAGON_SHA256}" "${DOWNLOAD_CACHE_DIRECTORY}" && \
-    mkdir -p /var/www/drupal/web/sites/all/assets/vendor && \
-    unzip "${DOWNLOAD_CACHE_DIRECTORY}/${OPENSEADRAGON_FILE}" -d /var/www/drupal/web/sites/all/assets/vendor && \
-    mv "/var/www/drupal/web/sites/all/assets/vendor/${OPENSEADRAGON_FILE%.zip}" /var/www/drupal/web/sites/all/assets/vendor/openseadragon && \
+    # Views Slideshow Dependencies
+    ## jquery.cycle 
+    JQUERY_CYCLE_FILE="jquery.cycle.all.js" && \
+    JQUERY_CYCLE_URL="https://malsup.github.io/${JQUERY_CYCLE_FILE}" && \
+    JQUERY_CYCLE_SHA256="58b44d975e1e1f0664d0fb8ab5b2918d08e9497324a021aa93de5894cdb586d4" && \
+    mkdir -p /var/www/drupal/web/libraries/jquery.cycle && \
+    download.sh --url "${JQUERY_CYCLE_URL}" --sha256 "${JQUERY_CYCLE_SHA256}" /var/www/drupal/web/libraries/jquery.cycle && \
+    ## jquery.hoverIntent
+    JQUERY_HOVER_INTENT_FILE="jquery.hoverIntent.js" && \
+    JQUERY_HOVER_INTENT_URL="https://github.com/raw/briancherne/jquery-hoverIntent/master/${JQUERY_HOVER_INTENT_FILE}" && \
+    JQUERY_HOVER_INTENT_SHA256="65f5f7e1298fe71f10290f4068df30b38a5df0106d6feb63210ddabcc67c3e59" && \
+    download.sh --url "${JQUERY_HOVER_INTENT_URL}" --sha256 "${JQUERY_HOVER_INTENT_SHA256}" "${DOWNLOAD_CACHE_DIRECTORY}" && \
+    mkdir -p /var/www/drupal/web/libraries/jquery.hoverIntent && \
+    cp "${DOWNLOAD_CACHE_DIRECTORY}/${JQUERY_HOVER_INTENT_FILE}" /var/www/drupal/web/libraries/jquery.hoverIntent && \
+    ## jquery.pause
+    JQUERY_PAUSE_FILE="jquery.pause.js" && \
+    JQUERY_PAUSE_URL="https://github.com/raw/tobia/Pause/master/${JQUERY_PAUSE_FILE}" && \
+    JQUERY_PAUSE_SHA256="39505a2a9fe36fce5b987f6804723d323ac86d0ed7220a5c12094f1d698fce33" && \
+    download.sh --url "${JQUERY_PAUSE_URL}" --sha256 "${JQUERY_PAUSE_SHA256}" "${DOWNLOAD_CACHE_DIRECTORY}" && \
+    mkdir -p /var/www/drupal/web/libraries/jquery.pause && \
+    cp "${DOWNLOAD_CACHE_DIRECTORY}/${JQUERY_PAUSE_FILE}" /var/www/drupal/web/libraries/jquery.pause && \
+    ## json2
+    JSON2_FILE="jquery.pause.js" && \
+    JSON2_URL="https://github.com/raw/tobia/Pause/master/${JSON2_FILE}" && \
+    JSON2_SHA256="39505a2a9fe36fce5b987f6804723d323ac86d0ed7220a5c12094f1d698fce33" && \
+    download.sh --url "${JSON2_URL}" --sha256 "${JSON2_SHA256}" "${DOWNLOAD_CACHE_DIRECTORY}" && \
+    mkdir -p /var/www/drupal/web/libraries/json2 && \
+    cp "${DOWNLOAD_CACHE_DIRECTORY}/${JSON2_FILE}" /var/www/drupal/web/libraries/json2 && \
+    # Openseadragon
     mkdir -p /var/www/drupal/web/sites/default/files/library-definitions && \
     cp /var/www/drupal/web/modules/contrib/openseadragon/openseadragon.json /var/www/drupal/web/sites/default/files/library-definitions && \
     chown -R nginx:nginx /var/www/drupal && \
@@ -63,5 +62,3 @@ RUN --mount=type=cache,target=/root/.composer/cache \
 FROM local/drupal:latest
 
 COPY --chown=nginx:nginx --from=composer /var/www/drupal /var/www/drupal
-
-COPY rootfs /

drupal/rootfs/etc/islandora/utilities.sh

@@ -77,7 +77,6 @@ function wait_for_required_services {
         wait_for_service "${site}" "SOLR"
         wait_for_service "${site}" "FCREPO"
         wait_for_service "${site}" "BROKER"
-        wait_for_service "${site}" "GEMINI"
         wait_for_service "${site}" "TRIPLESTORE"
     fi
 }
@@ -239,8 +238,8 @@ function install_site {
     fi
 
     # Ensure the files directory is writable by nginx, as when it is a new volume it is owned by root.
-	chown -R 100:101 "${files_directory}"
-	chmod -R ug+rw "${files_directory}"
+    chown -R 100:101 "${files_directory}"
+    chmod -R ug+rw "${files_directory}"
 
     # Allow changes to settings.php if it exists.
     if [[ -f "${site_directory}/settings.php" ]]; then
@@ -360,29 +359,31 @@ function configure_islandora_module {
     local broker_host=$(drupal_site_env "${site}" "BROKER_HOST")
     local broker_port=$(drupal_site_env "${site}" "BROKER_PORT")
     local broker_url="tcp://${broker_host}:${broker_port}"
-    local gemini_host=$(drupal_site_env "${site}" "GEMINI_HOST")
-    local gemini_port=$(drupal_site_env "${site}" "GEMINI_PORT")
-    local gemini_url="http://${gemini_host}:${gemini_port}"
 
-    drush -l "${site_url}" -y pm:enable islandora
+    drush -l "${site_url}" -y pm:enable islandora_core_feature 
     drush -l "${site_url}" -y config:set --input-format=yaml jsonld.settings remove_jsonld_format true
     drush -l "${site_url}" -y config:set --input-format=yaml islandora.settings broker_url "${broker_url}"
-    drush -l "${site_url}" -y config:set --input-format=yaml islandora.settings gemini_url "${gemini_url}"
-    drush -l "${site_url}" -y config:set --input-format=yaml islandora.settings gemini_pseudo_bundles.0 "islandora_object:node"
-    drush -l "${site_url}" -y config:set --input-format=yaml islandora.settings gemini_pseudo_bundles.1 "image:media"
-    drush -l "${site_url}" -y config:set --input-format=yaml islandora.settings gemini_pseudo_bundles.2 "file:media"
-    drush -l "${site_url}" -y config:set --input-format=yaml islandora.settings gemini_pseudo_bundles.3 "audio:media"
-    drush -l "${site_url}" -y config:set --input-format=yaml islandora.settings gemini_pseudo_bundles.4 "video:media"
+
+    if drush -l "${site_url}" role:list | grep -q fedoraadmin; then
+        echo "Fedora Admin role already exists.  No need to create it."
+    else
+        drush -l "${site_url}" role:create fedoraadmin fedoraAdmin
+    fi
+    drush -l "${site_url}" -y user:role:add fedoraadmin admin
 }
 
 # After enabling and importing features a number of configurations need to be updated.
 function configure_islandora_default_module {
+    if ! drush pm-list --pipe --type=module --status=enabled --no-core | grep -q islandora_defaults; then
+        echo "islandora_defaults is not installed.  Skipping configuration"
+        return 0
+    fi
+
     local site="${1}"; shift
     local site_url=$(drupal_site_env "${site}" "SITE_URL")
     local host=$(drupal_site_env "${site}" "SOLR_HOST")
     local port=$(drupal_site_env "${site}" "SOLR_PORT")
 
-    drush -l "${site_url}" -y user:role:add fedoraadmin admin
     drush -l "${site_url}" -y config:set search_api.server.default_solr_server backend_config.connector_config.host "${host}"
     drush -l "${site_url}" -y config:set search_api.server.default_solr_server backend_config.connector_config.port "${port}"
 }
@@ -403,9 +404,13 @@ function configure_search_api_solr_module {
 
 # Enables and sets carapace as the default theme.
 function set_carapace_default_theme {
+    if ! drush pm-list --pipe --type=theme --status=enabled --no-core | grep -q carapace; then
+        echo "carapace is not available. Skipping configuration."
+        return 0
+    fi
+
     local site="${1}"; shift
     local site_url=$(drupal_site_env "${site}" "SITE_URL")
-    drush -l "${site_url}" -y theme:enable carapace
     drush -l "${site_url}" -y config:set system.theme default carapace
 }
 
@@ -445,27 +450,41 @@ function create_solr_core {
 
 # Generate solr config and create a core for it.
 function create_solr_core_with_default_config {
+    if ! drush pm-list --pipe --type=module --status=enabled --no-core | grep -q search_api_solr; then
+        echo "search_api_solr is not installed.  Skipping core setup."
+        return 0
+    fi
+
     local site="${1}"; shift
     generate_solr_config "${site}"
     create_solr_core "${site}"
 }
 
 # Install matomo and configure.
 function configure_matomo_module {
+    if ! drush pm-list --pipe --type=module --status=enabled --no-core | grep -q matomo; then
+        echo "matomo is not installed.  Skipping configuration"
+        return 0
+    fi
+
     local site="${1}"; shift
     local site_url=$(drupal_site_env "${site}" "SITE_URL")
     local site_id=$(($(site_index "${site}")+1))
     local matomo_url=$(drupal_site_env "${site}" "MATOMO_URL")
     local matomo_http_url="http${matomo_url#https}"
 
-    drush -l "${site_url}" -y pm:enable matomo
     drush -l "${site_url}" -y config-set matomo.settings site_id "${site_id}"
     drush -l "${site_url}" -y config-set matomo.settings url_http "${matomo_http_url}"
     drush -l "${site_url}" -y config-set matomo.settings url_https "${matomo_url}"
 }
 
 # Configure Openseadragon to point use cantaloupe.
 function configure_openseadragon  {
+    if ! drush pm-list --pipe --type=module --status=enabled --no-core | grep -q openseadragon; then
+        echo "openseadragon is not installed.  Skipping configuration"
+        return 0
+    fi
+
     local site="${1}"; shift
     local site_url=$(drupal_site_env "${site}" "SITE_URL")
     local cantaloupe_url=$(drupal_site_env "${site}" "CANTALOUPE_URL")