Merge branch 'feature/metadata-search' into test/metadata-search

Cytomine-ULiege · Sep 25, 2023 · 4425e9f · 4425e9f
2 parents 23634e9 + b8d6ced
commit 4425e9f
Show file tree

Hide file tree

Showing 14 changed files with 495 additions and 116 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,10 +1,20 @@
+# Secrets should never be versioned.
+include:
+  - template: Security/Secret-Detection.gitlab-ci.yml
+
+
 stages:
   - prepare
   - test
   - build
   - publish
 
 variables:
+  DOCKER_IMAGE_NAME: 'cytomine/core'
+  ENTRYPOINT_SCRIPTS_VERSION: '1.3.0'
+  GRADLE_VERSION: '7.2-jdk17-alpine'
+  OPENJDK_VERSION: '17-slim-bullseye'
+
   # cache gradle working directory
   GRADLE_USER_HOME: /cache/.gradle
   POSTGRES_USER: docker
@@ -48,10 +58,27 @@ workflow:
     #    IS_OPEN_SOURCE_RELEASE_CANDIDATE: "false"
     - when: always
 
+lint-dockerfiles:
+  stage: prepare
+  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/hadolint/hadolint:latest-alpine
+  # Uncomment to silent some warnings if needed
+  #variables:
+  #  HADOLINT_OPT: --ignore DL3008
+  script:
+    - hadolint ${HADOLINT_OPT} docker/Dockerfile
+  rules:
+    - changes:
+        - docker/Dockerfile
+      when: always
+
+    # If the Dockerfile has not been changed, this job can be skipped
+    - when: never
+      allow_failure: true
+
 make-version-name:
   stage: prepare
   # regex check does not work with /bin/sh because of parenthesis so we need /bin/bash here
-  image: bash:5.2.15-alpine3.16
+  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/bash:5.2.15-alpine3.16
   script:
     #- if [[ "$CI_COMMIT_REF_NAME" =~ ^refs/tags/ ]] then
     #    echo "CM_VERSION=$CI_COMMIT_TAG" > .env;
@@ -94,89 +121,126 @@ run-junit:
       junit: build/test-results/test/**/TEST-*.xml
       # TODO integrate code coverage
 
-build-jar:
-  stage: build
-  image: docker:latest
-  rules:
-    - if: $CI_COMMIT_TAG
-  needs:
-    # no dependence on junit tests success. This allow for build
-    # stage jobs to run in parallel with tests. If test fails,
-    # publish jobs are aborted
-    - job: make-version-name
-      artifacts: true
-  script:
-    - docker build --build-arg CORE_VERSION=$CM_VERSION -t $DOCKER_TMP_JAR_IMAGE_NAME -f docker/Dockerfile --target jar-builder .
-    - docker run --name $CI_PIPELINE_ID-builder-container -t $DOCKER_TMP_JAR_IMAGE_NAME
-    - mkdir -p build/libs/
-    - docker cp $CI_PIPELINE_ID-builder-container:/app/build/libs/cytomine.jar build/libs/cytomine-$CM_VERSION.jar
-  after_script:
-    - docker rm $CI_PIPELINE_ID-builder-container || true
-    - docker rmi $DOCKER_TMP_JAR_IMAGE_NAME
-  artifacts:
-    when: on_success
-    expire_in: never
-    paths:
-      - build/libs/cytomine-*.jar
+#build-jar:
+#  stage: build
+#  image: docker:latest
+#  rules:
+#    - if: $CI_COMMIT_TAG
+#  needs:
+#    # no dependence on junit tests success. This allow for build
+#    # stage jobs to run in parallel with tests. If test fails,
+#    # publish jobs are aborted
+#    - job: make-version-name
+#      artifacts: true
+#  script:
+#    - docker build --build-arg CORE_VERSION=$CM_VERSION -t $DOCKER_TMP_JAR_IMAGE_NAME -f docker/Dockerfile --target jar-builder .
+#    - docker run --name $CI_PIPELINE_ID-builder-container -t $DOCKER_TMP_JAR_IMAGE_NAME
+#    - mkdir -p build/libs/
+#    - docker cp $CI_PIPELINE_ID-builder-container:/app/build/libs/cytomine.jar build/libs/cytomine-$CM_VERSION.jar
+#  after_script:
+#    - docker rm $CI_PIPELINE_ID-builder-container || true
+#    - docker rmi $DOCKER_TMP_JAR_IMAGE_NAME
+#  artifacts:
+#    when: on_success
+#    expire_in: never
+#    paths:
+#      - build/libs/cytomine-*.jar
+
+#build-docker-image:
+#  stage: build
+#  image: docker:latest
+#  needs:
+#    - job: build-jar
+#      artifacts: false
+#    - job: make-version-name
+#      artifacts: true
+#  rules:
+#    - if: $CI_COMMIT_TAG
+#  script:
+#    - docker build --build-arg CORE_VERSION=$CM_VERSION -t $DOCKER_TMP_IMAGE_NAME -f docker/Dockerfile .
+#
+#publish-docker-image:
+#  stage: publish
+#  image: docker:latest
+#  needs:
+#    - job: build-docker-image
+#      artifacts: false  # artifact are in docker cache (is it a safe assumption in a multi-runner env ?)
+#    - job: run-junit
+#      artifacts: false
+#    - job: make-version-name
+#      artifacts: true
+#  rules:
+#    - if: $CI_COMMIT_TAG
+#  script:
+#    - |
+#      if [ $IS_OPEN_SOURCE_RELEASE_CANDIDATE = "true" ]; then
+#        export DOCKER_IMAGE_NAME=cytomine/core
+#        export TARGET_CI_REGISTRY=$DOCKERHUB_URL
+#        export TARGET_CI_REGISTRY_USER=$DOCKERHUB_USER
+#        export TARGET_CI_REGISTRY_PASSWORD=$DOCKERHUB_PASSWORD
+#      else
+#        export DOCKER_IMAGE_NAME=$CI_REGISTRY_IMAGE
+#        export TARGET_CI_REGISTRY=$CI_REGISTRY
+#        export TARGET_CI_REGISTRY_USER=$CI_REGISTRY_USER
+#        export TARGET_CI_REGISTRY_PASSWORD=$CI_REGISTRY_PASSWORD
+#      fi
+#    - echo "Target > registry:$TARGET_CI_REGISTRY user:$TARGET_CI_REGISTRY_USER"
+#    - docker login -u $TARGET_CI_REGISTRY_USER -p $TARGET_CI_REGISTRY_PASSWORD $TARGET_CI_REGISTRY
+#    - export DOCKER_IMAGE_NAME_FULL=$DOCKER_IMAGE_NAME:$CM_VERSION
+#    - docker tag $DOCKER_TMP_IMAGE_NAME $DOCKER_IMAGE_NAME_FULL
+#    - docker push $DOCKER_IMAGE_NAME_FULL
+#  after_script:
+#    # cleaning up images
+#    - docker rmi $DOCKER_IMAGE_NAME_FULL $DOCKER_TMP_IMAGE_NAME
 
-build-docker-image:
+
+build-push-docker-image:
   stage: build
-  image: docker:latest
+  image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/docker:latest
   needs:
-    - job: build-jar
-      artifacts: false
     - job: make-version-name
       artifacts: true
   rules:
     - if: $CI_COMMIT_TAG
   script:
-    - docker build --build-arg CORE_VERSION=$CM_VERSION -t $DOCKER_TMP_IMAGE_NAME -f docker/Dockerfile .
+    - echo "Log in to Gitlab docker registry"
+    - mkdir -p ~/.docker
+    - echo ${DOCKER_AUTH_CONFIG} > ~/.docker/config.json
 
-publish-docker-image:
-  stage: publish
-  image: docker:latest
-  needs:
-    - job: build-docker-image
-      artifacts: false  # artifact are in docker cache (is it a safe assumption in a multi-runner env ?)
-    - job: run-junit
-      artifacts: false
-    - job: make-version-name
-      artifacts: true
-  rules:
-    - if: $CI_COMMIT_TAG
-  script:
+    - echo "Build the docker image ${DOCKER_IMAGE_NAME}:${CM_VERSION}"
     - |
-      if [ $IS_OPEN_SOURCE_RELEASE_CANDIDATE = "true" ]; then
-        export DOCKER_IMAGE_NAME=cytomine/core
-        export TARGET_CI_REGISTRY=$DOCKERHUB_URL
-        export TARGET_CI_REGISTRY_USER=$DOCKERHUB_USER
-        export TARGET_CI_REGISTRY_PASSWORD=$DOCKERHUB_PASSWORD
-      else
-        export DOCKER_IMAGE_NAME=$CI_REGISTRY_IMAGE
-        export TARGET_CI_REGISTRY=$CI_REGISTRY
-        export TARGET_CI_REGISTRY_USER=$CI_REGISTRY_USER
-        export TARGET_CI_REGISTRY_PASSWORD=$CI_REGISTRY_PASSWORD
-      fi
-    - echo "Target > registry:$TARGET_CI_REGISTRY user:$TARGET_CI_REGISTRY_USER"
-    - docker login -u $TARGET_CI_REGISTRY_USER -p $TARGET_CI_REGISTRY_PASSWORD $TARGET_CI_REGISTRY
-    - export DOCKER_IMAGE_NAME_FULL=$DOCKER_IMAGE_NAME:$CM_VERSION
-    - docker tag $DOCKER_TMP_IMAGE_NAME $DOCKER_IMAGE_NAME_FULL
-    - docker push $DOCKER_IMAGE_NAME_FULL
-  after_script:
-    # cleaning up images
-    - docker rmi $DOCKER_IMAGE_NAME_FULL $DOCKER_TMP_IMAGE_NAME
-
-publish-jar:
-  stage: publish
-  image: alpine:latest
-  rules:
-    - if: $CI_COMMIT_TAG
-  needs:
-    - job: make-version-name
-      artifacts: true
-    - job: build-jar
-      artifacts: true
-    - job: run-junit
-      artifacts: false
-  script:
-    - echo "TODO upload somehwere"
+      docker build \
+        --build-arg CORE_VERSION=${CM_VERSION} \
+        --build-arg CORE_REVISION=${CI_COMMIT_SHORT_SHA} \
+        --build-arg ENTRYPOINT_SCRIPTS_VERSION=${ENTRYPOINT_SCRIPTS_VERSION} \
+        --build-arg GRADLE_VERSION=${GRADLE_VERSION} \
+        --build-arg OPENJDK_VERSION=${OPENJDK_VERSION} \
+        -t ${DOCKER_IMAGE_NAME}:${CM_VERSION} \
+        -t ${DOCKER_IMAGE_NAME}:latest \
+        -f docker/Dockerfile \
+        .
+
+    # Use credential helper (see https://docs.docker.com/engine/reference/commandline/login/#credentials-store)
+    - mkdir -p $HOME/.docker && echo $DOCKER_HUB_AUTH_CONFIG > $HOME/.docker/config.json
+    - echo "Registry credentials configured at $HOME/.docker/config.json"
+
+    - echo "Pushing docker image ${DOCKER_IMAGE_NAME}:${CM_VERSION} and ${DOCKER_IMAGE_NAME}:latest"
+    - docker push ${DOCKER_IMAGE_NAME}:${CM_VERSION}
+    - docker push ${DOCKER_IMAGE_NAME}:latest
+    - echo "Successfully pushed docker image ${DOCKER_IMAGE_NAME}:${CM_VERSION} and ${DOCKER_IMAGE_NAME}:latest"
+
+
+#publish-jar:
+#  stage: publish
+#  image: alpine:latest
+#  rules:
+#    - if: $CI_COMMIT_TAG
+#  needs:
+#    - job: make-version-name
+#      artifacts: true
+#    - job: build-jar
+#      artifacts: true
+#    - job: run-junit
+#      artifacts: false
+#  script:
+#    - echo "TODO upload somehwere"
diff --git a/build.gradle b/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-	id 'org.springframework.boot' version '2.6.6'
+	id 'org.springframework.boot' version '2.7.10'
 	id 'io.spring.dependency-management' version '1.0.11.RELEASE'
 	id 'java'
 	id 'jacoco'
@@ -95,6 +95,9 @@ dependencies {
 	implementation 'org.apache.poi:poi-ooxml:5.2.3'
 
 
+	// Elasticsearch
+	implementation('jakarta.json:jakarta.json-api:2.0.1')
+	implementation('org.springframework.boot:spring-boot-starter-data-elasticsearch')
 }
 
 test {

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,10 +1,13 @@
 ARG CORE_VERSION
-ARG SCRIPTS_REPO_TAG
+ARG CORE_REVISION
+ARG ENTRYPOINT_SCRIPTS_VERSION=1.3.0
+ARG GRADLE_VERSION=7.2-jdk17-alpine
+ARG OPENJDK_VERSION=17-slim-bullseye
 
 
 #######################################################################################
 # Stage: core dependencies download via gradle
-FROM gradle:7.2-jdk17-alpine AS deps-downloader
+FROM gradle:${GRADLE_VERSION} AS deps-downloader
 
 # We first copy the build.gradle file and the binaries stored in the source repository.
 # This way, we retrieve all gradle dependencies at the beginning. All these steps will be
@@ -20,7 +23,7 @@ RUN gradle clean build --no-daemon --console=verbose
 
 #######################################################################################
 ## Stage: building the core jar file
-FROM gradle:7.2-jdk17-alpine AS jar-builder
+FROM gradle:${GRADLE_VERSION} AS jar-builder
 
 ENV GRADLE_USER_HOME=/opt/gradle/.gradle
 COPY --from=deps-downloader /opt/gradle/.gradle /opt/gradle/.gradle
@@ -32,38 +35,29 @@ COPY ../build.gradle /app/build.gradle
 ARG CORE_VERSION
 ENV CORE_VERSION=$CORE_VERSION
 
-RUN sed -i -- 's/version: 0.0.0/version: '$CORE_VERSION'/g' /app/src/main/resources/application.yml
-
-RUN gradle bootJar --console=verbose
+RUN sed -i -- 's/version: 0.0.0/version: '$CORE_VERSION'/g' /app/src/main/resources/application.yml && \
+    gradle bootJar --console=verbose
 
 #######################################################################################
-## Stage: downloading provisioning scripts
-FROM alpine/git:2.36.3 as scripts-downloader
-
-ARG SCRIPTS_REPO_TAG="latest"
-ARG SCRIPT_REPO_URL="https://github.com/cytomine/cytomine-docker-entrypoint-scripts.git"
-
-WORKDIR /root
-
-RUN mkdir scripts
-RUN git clone $SCRIPT_REPO_URL /root/scripts \
-    && cd /root/scripts \
-    && git checkout tags/$SCRIPTS_REPO_TAG
+## Stage: entrypoint script. Use a multi-stage because COPY --from cannot interpolate variables
+FROM cytomine/entrypoint-scripts:${ENTRYPOINT_SCRIPTS_VERSION} as entrypoint-scripts
 
 #######################################################################################
 ## Stage: Cytomine core development image
-FROM gradle:7.2-jdk17-alpine AS development
+FROM gradle:${GRADLE_VERSION} AS development
 
-RUN apk update && apk add gettext
+RUN apk update && \
+    apk add --no-cache \
+    gettext~=0.21
 
 ENV GRADLE_USER_HOME=/opt/gradle/.gradle
 COPY --from=deps-downloader /opt/gradle/.gradle /opt/gradle/.gradle
 
 # startup scripts
 RUN mkdir /docker-entrypoint-cytomine.d/
-COPY --from=scripts-downloader --chmod=774 /root/scripts/cytomine-entrypoint.sh /usr/local/bin/
-COPY --from=scripts-downloader --chmod=774 /root/scripts/envsubst-on-templates-and-move.sh /docker-entrypoint-cytomine.d/500-envsubst-on-templates-and-move.sh
-COPY --from=scripts-downloader --chmod=774 /root/scripts/configure-etc-hosts-reverse-proxy.sh /docker-entrypoint-cytomine.d/750-configure-etc-hosts-reverse-proxy.sh
+COPY --from=entrypoint-scripts --chmod=774 /cytomine-entrypoint.sh /usr/local/bin/
+COPY --from=entrypoint-scripts --chmod=774 /envsubst-on-templates-and-move.sh /docker-entrypoint-cytomine.d/500-envsubst-on-templates-and-move.sh
+COPY --from=entrypoint-scripts --chmod=774 /configure-etc-hosts-reverse-proxy.sh /docker-entrypoint-cytomine.d/750-configure-etc-hosts-reverse-proxy.sh
 
 # core base folder must be mounted in run command or compose file
 EXPOSE 5005
@@ -74,25 +68,43 @@ CMD ["gradle", ":bootRun", "--debug-jvm"]
 
 #######################################################################################
 ## Stage: Cytomine core image
-FROM openjdk:17-slim-bullseye as production
+FROM openjdk:${OPENJDK_VERSION} as production
 
-# base librairies and configuration
-RUN apt-get update -y && apt-get -y install logrotate
+ARG CORE_VERSION
+ARG CORE_REVISION
+ARG ENTRYPOINT_SCRIPTS_VERSION
+ARG GRADLE_VERSION
+ARG OPENJDK_VERSION
+
+LABEL org.opencontainers.image.authors="support@cytomine.com" \
+      org.opencontainers.image.url="https://www.cytomine.org/" \
+      org.opencontainers.image.documentation="https://doc.cytomine.org/" \
+      org.opencontainers.image.source="https://github.com/cytomine/Cytomine-core-spring" \
+      org.opencontainers.image.vendor="Cytomine Corporation SA" \
+      org.opencontainers.image.version="${CORE_VERSION}" \
+      org.opencontainers.image.revision="${CORE_REVISION}" \
+      org.opencontainers.image.deps.openjdk.version="${OPENJDK_VERSION}" \
+      org.opencontainers.image.deps.gradle.version="${GRADLE_VERSION}" \
+      org.opencontainers.image.deps.entrypoint.scripts.version="${ENTRYPOINT_SCRIPTS_VERSION}"
 
-RUN sed -i "/su root syslog/c\su root root" /etc/logrotate.conf
-ENV LANG C.UTF-8
 ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
 
-# tomcat configurations
-RUN apt-get update -y && apt-get install --no-install-recommends --no-install-suggests -y logrotate gettext
+# base librairies and configuration
+RUN apt-get update -y \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+      logrotate=3.18* \
+      gettext=0.21* \
+    && rm -rf /var/lib/apt/lists/* \
+    && sed -i "/su root syslog/c\su root root" /etc/logrotate.conf
 
 COPY --from=jar-builder /app/build/libs/cytomine.jar /app/cytomine.jar
 
 # entrypoint scripts
 RUN mkdir /docker-entrypoint-cytomine.d/
-COPY --from=scripts-downloader --chmod=774 /root/scripts/cytomine-entrypoint.sh /usr/local/bin/
-COPY --from=scripts-downloader --chmod=774 /root/scripts/envsubst-on-templates-and-move.sh /docker-entrypoint-cytomine.d/500-envsubst-on-templates-and-move.sh
-COPY --from=scripts-downloader --chmod=774 /root/scripts/configure-etc-hosts-reverse-proxy.sh /docker-entrypoint-cytomine.d/750-configure-etc-hosts-reverse-proxy.sh
+COPY --from=entrypoint-scripts --chmod=774 /cytomine-entrypoint.sh /usr/local/bin/
+COPY --from=entrypoint-scripts --chmod=774 /envsubst-on-templates-and-move.sh /docker-entrypoint-cytomine.d/500-envsubst-on-templates-and-move.sh
+COPY --from=entrypoint-scripts --chmod=774 /configure-etc-hosts-reverse-proxy.sh /docker-entrypoint-cytomine.d/750-configure-etc-hosts-reverse-proxy.sh
 COPY --chmod=774 ./scripts/generate-meta-prefix-file.sh /docker-entrypoint-cytomine.d/875-generate-meta-prefix-file.sh
 
 WORKDIR /app