From 9034ce8dc75fab25c2a0a3aa239f10f6311c1fa0 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 15:22:30 +0100 Subject: [PATCH 01/14] Kcat builded within Product Image --- kafka/Dockerfile | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index c70a45c7..88e7e3e0 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -1,3 +1,14 @@ +FROM docker.stackable.tech/stackable/javabase:11-stackable0.2.2@sha256:7929833412c331fc23cde0e23ca730d652c0be61a8a69c8a82b2af937a3fbd4e AS builder + +RUN microdnf install zlib-devel openssl-devel cyrus-sasl-devel libcurl-devel +RUN microdnf install tar which wget zlib gcc-c++ make cmake + +WORKDIR /stackable +RUN curl -L -O https://github.com/edenhill/kcat/archive/refs/tags/1.7.0.tar.gz \ + && tar xvfz 1.7.0.tar.gz \ + && cd kcat-1.7.0 \ + && ./bootstrap.sh + FROM docker.stackable.tech/stackable/java-base:11-stackable0.2.2@sha256:7929833412c331fc23cde0e23ca730d652c0be61a8a69c8a82b2af937a3fbd4e ARG PRODUCT @@ -63,5 +74,7 @@ COPY shared/log4shell_scanner /bin/log4shell_scanner RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT} # === +COPY --from=builder /stackable/kcat-1.7.0/kcat /stackable/kcat + WORKDIR /stackable/kafka -CMD ["bin/kafka-server-start.sh", "/stackable/kafka/config/server.properties"] +CMD ["bin/kafka-server-start.sh", "/stackable/kafka/config/server.properties"] \ No newline at end of file From daf8f0fea1876d93a9d7d7428f0be7605b844991 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 16:36:21 +0100 Subject: [PATCH 02/14] Added bcrypt to nifi image --- nifi/Dockerfile | 4 ++++ ...le-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar | Bin 2 files changed, 4 insertions(+) rename {tools => nifi}/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar (100%) diff --git a/nifi/Dockerfile b/nifi/Dockerfile index b2dd988d..136a5f0f 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -19,6 +19,7 @@ RUN microdnf update && \ microdnf install shadow-utils && \ microdnf clean all + COPY nifi/stackable /stackable COPY nifi/licenses /licenses @@ -55,5 +56,8 @@ COPY shared/log4shell_scanner /bin/log4shell_scanner RUN /bin/log4shell_scanner s /stackable/nifi-${PRODUCT} # === +# The bcrypt tool is needed by NiFi to locally encrypt the admin password that is mounted as a secret in cleartext +COPY nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar /bin/stackable-bcrypt.jar + WORKDIR /stackable/nifi CMD ["bin/nifi.sh", "run"] diff --git a/tools/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar b/nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar similarity index 100% rename from tools/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar rename to nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar From 25d16d3a304e3a96a4200484563848b73e76cfa6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 16:38:26 +0100 Subject: [PATCH 03/14] Added nifi reporting task python script to nifi image --- nifi/Dockerfile | 2 + tools/python/create_nifi_reporting_task.py | 151 --------------------- tools/python/requirements.txt | 1 - 3 files changed, 2 insertions(+), 152 deletions(-) delete mode 100755 tools/python/create_nifi_reporting_task.py delete mode 100644 tools/python/requirements.txt diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 136a5f0f..72924535 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -58,6 +58,8 @@ RUN /bin/log4shell_scanner s /stackable/nifi-${PRODUCT} # The bcrypt tool is needed by NiFi to locally encrypt the admin password that is mounted as a secret in cleartext COPY nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar /bin/stackable-bcrypt.jar +# add all python scripts +COPY nifi/python /stackable/python WORKDIR /stackable/nifi CMD ["bin/nifi.sh", "run"] diff --git a/tools/python/create_nifi_reporting_task.py b/tools/python/create_nifi_reporting_task.py deleted file mode 100755 index 7cd801c3..00000000 --- a/tools/python/create_nifi_reporting_task.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 -"""This is a script to create a ReportingTask in NiFi via the REST API""" -import sys -import argparse -import nipyapi - -# no stack trace -sys.tracebacklimit = 0 - - -def init(url: str, username: str, password: str, ca_file: str): - """Initialize authenticated connection to NiFi""" - nipyapi.config.nifi_config.host = url - nipyapi.security.set_service_ssl_context(service='nifi', ca_file=ca_file) - - try: - nipyapi.security.service_login(service='nifi', username=username, password=password) - print("Successfully authenticated and established connection with [%s]" % url) - except Exception as ex: - raise Exception("Failed to connect to {}: {}".format(url, str(ex))) from None - - -def find_reporting_task(name: str, port: str): - """Find a ReportingTask via its name and port""" - flow_api = nipyapi.nifi.apis.flow_api.FlowApi() - - try: - reporting_tasks = flow_api.get_reporting_tasks().reporting_tasks - except Exception as ex: - raise Exception("Failed to retrieve ReportingTask[{}/{}]: {}" - .format(name, port, str(ex))) from None - - for task in reporting_tasks: - task_dict = task.to_dict() - task_component = task_dict["component"] - task_name = task_component["name"] - task_port = task_component["properties"]["prometheus-reporting-task-metrics-endpoint-port"] - if task_name == name and task_port == port: - return task - - return None - - -def create_reporting_task(name: str, port: str, version: str): - """Create a ReportingTask""" - task = nipyapi.nifi.models.reporting_task_entity.ReportingTaskEntity( - revision=nipyapi.nifi.models.revision_dto.RevisionDTO(version=0), - disconnected_node_acknowledged=False, - component=nipyapi.nifi.models.reporting_task_dto.ReportingTaskDTO( - name=name, - type="org.apache.nifi.reporting.prometheus.PrometheusReportingTask", - bundle=nipyapi.nifi.models.bundle_dto.BundleDTO( - group="org.apache.nifi", - artifact="nifi-prometheus-nar", - version=version - ), - properties={ - "prometheus-reporting-task-metrics-endpoint-port": port, - "prometheus-reporting-task-metrics-send-jvm": True - } - ) - ) - - controller_api = nipyapi.nifi.apis.controller_api.ControllerApi() - - try: - return controller_api.create_reporting_task(body=task) - except Exception as ex: - raise Exception("Failed to create reporting task: {}".format(str(ex))) from None - - -def get_reporting_task_name(task): - """Return the ReportingTask name""" - task_dict = task.to_dict() - return task_dict["component"]["name"] - - -def get_revision_version(task): - """Return the ReportingTask revision version""" - task_dict = task.to_dict() - return task_dict["revision"]["version"] - - -def is_reporting_task_running(task): - """Check if the the ReportingTask is already running""" - task_dict = task.to_dict() - return task_dict["component"]["state"] == "RUNNING" - - -def set_reporting_task_running(task): - """Set ReportingTask to RUNNING""" - reporting_task_api = nipyapi.nifi.apis.reporting_tasks_api.ReportingTasksApi() - - state = { - "revision": { - "version": get_revision_version(task) - }, - "disconnected_node_acknowledged": False, - "state": "RUNNING" - } - - try: - return reporting_task_api.update_run_status(id=task.id, body=state) - except Exception as ex: - raise Exception("Failed to set ReportingTask [{}] to RUNNING: {}" - .format(task.id, str(ex))) from None - - -def main(): - """Main method with cli argument parsing and ReportingTask logic""" - # Construct an argument parser - all_args = argparse.ArgumentParser() - # Add arguments to the parser - all_args.add_argument("-n", "--nifi_api_url", required=True, - help="The NiFi node url to connect to.") - all_args.add_argument("-u", "--username", required=True, - help="Username to connect as.") - all_args.add_argument("-p", "--password", required=True, - help="Password for the user.") - all_args.add_argument("-v", "--nifi_version", required=True, - help="The NiFi product version.") - all_args.add_argument("-c", "--cert", required=True, - help="The path to the trusted certificate authority that " - "signed our expected certificates.") - all_args.add_argument("-m", "--metrics_port", required=True, - help="Metrics port to be set in the ReportingTask.") - all_args.add_argument("-t", "--task_name", required=False, - default="StackablePrometheusReportingTask", - help="The name of ReportingTask to create or activate.") - args = vars(all_args.parse_args()) - - task_name = args["task_name"] - port = args["metrics_port"] - - init(args["nifi_api_url"], args["username"], args["password"], args["cert"]) - - reporting_task = find_reporting_task(name=task_name, port=port) - - if reporting_task is None: - reporting_task = create_reporting_task(name=task_name, port=port, - version=args["nifi_version"]) - print(get_reporting_task_name(task=reporting_task) + " [%s] -> CREATED" % reporting_task.id) - - if not is_reporting_task_running(task=reporting_task): - reporting_task = set_reporting_task_running(task=reporting_task) - - print(get_reporting_task_name(task=reporting_task) + " [%s] -> RUNNING" % reporting_task.id) - - -if __name__ == '__main__': - main() diff --git a/tools/python/requirements.txt b/tools/python/requirements.txt deleted file mode 100644 index 788eb166..00000000 --- a/tools/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -nipyapi==0.19.1 \ No newline at end of file From 5acf614ac1572e384375cc4d001a5ac10e0ba1bc Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 16:38:49 +0100 Subject: [PATCH 04/14] Moved python scripts to nifi folder --- nifi/python/create_nifi_reporting_task.py | 151 ++++++++++++++++++++++ nifi/python/requirements.txt | 1 + 2 files changed, 152 insertions(+) create mode 100755 nifi/python/create_nifi_reporting_task.py create mode 100644 nifi/python/requirements.txt diff --git a/nifi/python/create_nifi_reporting_task.py b/nifi/python/create_nifi_reporting_task.py new file mode 100755 index 00000000..7cd801c3 --- /dev/null +++ b/nifi/python/create_nifi_reporting_task.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""This is a script to create a ReportingTask in NiFi via the REST API""" +import sys +import argparse +import nipyapi + +# no stack trace +sys.tracebacklimit = 0 + + +def init(url: str, username: str, password: str, ca_file: str): + """Initialize authenticated connection to NiFi""" + nipyapi.config.nifi_config.host = url + nipyapi.security.set_service_ssl_context(service='nifi', ca_file=ca_file) + + try: + nipyapi.security.service_login(service='nifi', username=username, password=password) + print("Successfully authenticated and established connection with [%s]" % url) + except Exception as ex: + raise Exception("Failed to connect to {}: {}".format(url, str(ex))) from None + + +def find_reporting_task(name: str, port: str): + """Find a ReportingTask via its name and port""" + flow_api = nipyapi.nifi.apis.flow_api.FlowApi() + + try: + reporting_tasks = flow_api.get_reporting_tasks().reporting_tasks + except Exception as ex: + raise Exception("Failed to retrieve ReportingTask[{}/{}]: {}" + .format(name, port, str(ex))) from None + + for task in reporting_tasks: + task_dict = task.to_dict() + task_component = task_dict["component"] + task_name = task_component["name"] + task_port = task_component["properties"]["prometheus-reporting-task-metrics-endpoint-port"] + if task_name == name and task_port == port: + return task + + return None + + +def create_reporting_task(name: str, port: str, version: str): + """Create a ReportingTask""" + task = nipyapi.nifi.models.reporting_task_entity.ReportingTaskEntity( + revision=nipyapi.nifi.models.revision_dto.RevisionDTO(version=0), + disconnected_node_acknowledged=False, + component=nipyapi.nifi.models.reporting_task_dto.ReportingTaskDTO( + name=name, + type="org.apache.nifi.reporting.prometheus.PrometheusReportingTask", + bundle=nipyapi.nifi.models.bundle_dto.BundleDTO( + group="org.apache.nifi", + artifact="nifi-prometheus-nar", + version=version + ), + properties={ + "prometheus-reporting-task-metrics-endpoint-port": port, + "prometheus-reporting-task-metrics-send-jvm": True + } + ) + ) + + controller_api = nipyapi.nifi.apis.controller_api.ControllerApi() + + try: + return controller_api.create_reporting_task(body=task) + except Exception as ex: + raise Exception("Failed to create reporting task: {}".format(str(ex))) from None + + +def get_reporting_task_name(task): + """Return the ReportingTask name""" + task_dict = task.to_dict() + return task_dict["component"]["name"] + + +def get_revision_version(task): + """Return the ReportingTask revision version""" + task_dict = task.to_dict() + return task_dict["revision"]["version"] + + +def is_reporting_task_running(task): + """Check if the the ReportingTask is already running""" + task_dict = task.to_dict() + return task_dict["component"]["state"] == "RUNNING" + + +def set_reporting_task_running(task): + """Set ReportingTask to RUNNING""" + reporting_task_api = nipyapi.nifi.apis.reporting_tasks_api.ReportingTasksApi() + + state = { + "revision": { + "version": get_revision_version(task) + }, + "disconnected_node_acknowledged": False, + "state": "RUNNING" + } + + try: + return reporting_task_api.update_run_status(id=task.id, body=state) + except Exception as ex: + raise Exception("Failed to set ReportingTask [{}] to RUNNING: {}" + .format(task.id, str(ex))) from None + + +def main(): + """Main method with cli argument parsing and ReportingTask logic""" + # Construct an argument parser + all_args = argparse.ArgumentParser() + # Add arguments to the parser + all_args.add_argument("-n", "--nifi_api_url", required=True, + help="The NiFi node url to connect to.") + all_args.add_argument("-u", "--username", required=True, + help="Username to connect as.") + all_args.add_argument("-p", "--password", required=True, + help="Password for the user.") + all_args.add_argument("-v", "--nifi_version", required=True, + help="The NiFi product version.") + all_args.add_argument("-c", "--cert", required=True, + help="The path to the trusted certificate authority that " + "signed our expected certificates.") + all_args.add_argument("-m", "--metrics_port", required=True, + help="Metrics port to be set in the ReportingTask.") + all_args.add_argument("-t", "--task_name", required=False, + default="StackablePrometheusReportingTask", + help="The name of ReportingTask to create or activate.") + args = vars(all_args.parse_args()) + + task_name = args["task_name"] + port = args["metrics_port"] + + init(args["nifi_api_url"], args["username"], args["password"], args["cert"]) + + reporting_task = find_reporting_task(name=task_name, port=port) + + if reporting_task is None: + reporting_task = create_reporting_task(name=task_name, port=port, + version=args["nifi_version"]) + print(get_reporting_task_name(task=reporting_task) + " [%s] -> CREATED" % reporting_task.id) + + if not is_reporting_task_running(task=reporting_task): + reporting_task = set_reporting_task_running(task=reporting_task) + + print(get_reporting_task_name(task=reporting_task) + " [%s] -> RUNNING" % reporting_task.id) + + +if __name__ == '__main__': + main() diff --git a/nifi/python/requirements.txt b/nifi/python/requirements.txt new file mode 100644 index 00000000..788eb166 --- /dev/null +++ b/nifi/python/requirements.txt @@ -0,0 +1 @@ +nipyapi==0.19.1 \ No newline at end of file From 4f165c95b9a7d458e98ef0a2ec15892fc27d5fd3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 17:34:40 +0100 Subject: [PATCH 05/14] Added openssl to Nifi --- nifi/Dockerfile | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 72924535..332657b1 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.stackable.tech/stackable/java-base:11-stackable0.2.2@sha256:7929833412c331fc23cde0e23ca730d652c0be61a8a69c8a82b2af937a3fbd4e +FROM docker.stackable.tech/stackable-experimental/java-base:11-stackable0.2.2@sha256:7929833412c331fc23cde0e23ca730d652c0be61a8a69c8a82b2af937a3fbd4e ARG PRODUCT ARG RELEASE="1" @@ -15,15 +15,23 @@ LABEL name="Apache NiFi" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install tar gzip zip && \ + microdnf install tar gzip zip openssl && \ + microdnf install python3-devel python3-pip python3-setuptools \ + gcc && \ microdnf install shadow-utils && \ microdnf clean all +# The bcrypt tool is needed by NiFi to locally encrypt the admin password that is mounted as a secret in cleartext +COPY nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar /bin/stackable-bcrypt.jar +# add all python scripts +COPY nifi/python /stackable/python + COPY nifi/stackable /stackable COPY nifi/licenses /licenses -RUN groupadd -r stackable --gid=1000 && \ +RUN pip3 install --no-cache-dir -r /stackable/python/requirements.txt && \ + groupadd -r stackable --gid=1000 && \ useradd -r -g stackable --uid=1000 stackable && \ chown -R stackable:stackable /stackable @@ -56,10 +64,5 @@ COPY shared/log4shell_scanner /bin/log4shell_scanner RUN /bin/log4shell_scanner s /stackable/nifi-${PRODUCT} # === -# The bcrypt tool is needed by NiFi to locally encrypt the admin password that is mounted as a secret in cleartext -COPY nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar /bin/stackable-bcrypt.jar -# add all python scripts -COPY nifi/python /stackable/python - WORKDIR /stackable/nifi CMD ["bin/nifi.sh", "run"] From 67b925f0489734ae52310cc431d095826508a757 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 17:37:18 +0100 Subject: [PATCH 06/14] Added openssl to Kafka --- kafka/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 88e7e3e0..fbf31f36 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -29,7 +29,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ microdnf install tar gzip zip && \ - microdnf install shadow-utils && \ + microdnf install shadow-utils openssl && \ microdnf clean all COPY kafka/stackable /stackable From 38ff515902b3634c1448170f8286ae3ee70d047b Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 18:01:11 +0100 Subject: [PATCH 07/14] Added openssl to trino and druid --- druid/Dockerfile | 2 +- trino/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index 95c59686..b64690c7 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -18,7 +18,7 @@ LABEL name="Apache Druid" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install findutils && \ + microdnf install findutils openssl && \ microdnf install tar gzip zip && \ microdnf install shadow-utils && \ microdnf clean all diff --git a/trino/Dockerfile b/trino/Dockerfile index 793b96f5..e06a6c34 100644 --- a/trino/Dockerfile +++ b/trino/Dockerfile @@ -18,7 +18,7 @@ LABEL name="Trino" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install tar gzip zip python3 && \ + microdnf install tar gzip zip python3 openssl && \ microdnf install shadow-utils && \ microdnf clean all From af64b5029db77657f021406adf8d48a9758dc1d6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Thu, 1 Dec 2022 18:36:20 +0100 Subject: [PATCH 08/14] Added libxml2 and libxslt to Nifi --- nifi/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 332657b1..fe61cf9c 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -15,7 +15,7 @@ LABEL name="Apache NiFi" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install tar gzip zip openssl && \ + microdnf install tar gzip zip openssl libxslt-devel libxml2-devel&& \ microdnf install python3-devel python3-pip python3-setuptools \ gcc && \ microdnf install shadow-utils && \ From f5d6c6b10662538f9b2f2233d08c4712273aa6d2 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 2 Dec 2022 08:50:41 +0100 Subject: [PATCH 09/14] Solve Hadolint --- kafka/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index fbf31f36..ea60a19a 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -1,7 +1,8 @@ FROM docker.stackable.tech/stackable/javabase:11-stackable0.2.2@sha256:7929833412c331fc23cde0e23ca730d652c0be61a8a69c8a82b2af937a3fbd4e AS builder -RUN microdnf install zlib-devel openssl-devel cyrus-sasl-devel libcurl-devel -RUN microdnf install tar which wget zlib gcc-c++ make cmake +RUN microdnf install -y zlib-devel openssl-devel cyrus-sasl-devel libcurl-devel && \ + microdnf install -y tar which wget zlib gcc-c++ make cmake && \ + microdnf clean all WORKDIR /stackable RUN curl -L -O https://github.com/edenhill/kcat/archive/refs/tags/1.7.0.tar.gz \ From 5a935d9c11d1268097713f9ce6e24cb82d7acff3 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Fri, 2 Dec 2022 08:53:00 +0100 Subject: [PATCH 10/14] Switched back to stackable, leftover from testing --- nifi/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi/Dockerfile b/nifi/Dockerfile index fe61cf9c..1953724a 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.stackable.tech/stackable-experimental/java-base:11-stackable0.2.2@sha256:7929833412c331fc23cde0e23ca730d652c0be61a8a69c8a82b2af937a3fbd4e +FROM docker.stackable.tech/stackable/java-base:11-stackable0.2.2@sha256:7929833412c331fc23cde0e23ca730d652c0be61a8a69c8a82b2af937a3fbd4e ARG PRODUCT ARG RELEASE="1" From 3cfd31d5eb33deba29b75b19c02eebab80788677 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 13 Dec 2022 12:36:28 +0100 Subject: [PATCH 11/14] Reduce NiFi image size --- nifi/Dockerfile | 30 ++++++++++++------------------ nifi/python/requirements.txt | 1 - 2 files changed, 12 insertions(+), 19 deletions(-) delete mode 100644 nifi/python/requirements.txt diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 1953724a..9bcf0fc0 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -15,29 +15,23 @@ LABEL name="Apache NiFi" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install tar gzip zip openssl libxslt-devel libxml2-devel&& \ - microdnf install python3-devel python3-pip python3-setuptools \ - gcc && \ - microdnf install shadow-utils && \ + microdnf install tar gzip zip openssl libxslt-devel libxml2-devel gcc shadow-utils && \ + microdnf install python3-devel python3-pip python3-setuptools && \ microdnf clean all - -# The bcrypt tool is needed by NiFi to locally encrypt the admin password that is mounted as a secret in cleartext -COPY nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar /bin/stackable-bcrypt.jar -# add all python scripts -COPY nifi/python /stackable/python - -COPY nifi/stackable /stackable -COPY nifi/licenses /licenses - -RUN pip3 install --no-cache-dir -r /stackable/python/requirements.txt && \ - groupadd -r stackable --gid=1000 && \ - useradd -r -g stackable --uid=1000 stackable && \ - chown -R stackable:stackable /stackable +RUN pip3 install --no-cache-dir nipyapi==0.19.1 +RUN groupadd -r stackable --gid=1000 && \ + useradd -r -g stackable --uid=1000 stackable USER stackable - WORKDIR /stackable + +# The bcrypt tool is needed by NiFi to locally encrypt the admin password that is mounted as a secret in cleartext +COPY --chown=stackable:stackable nifi/bin/stackable-bcrypt-1.0-SNAPSHOT-jar-with-dependencies.jar /bin/stackable-bcrypt.jar +COPY --chown=stackable:stackable nifi/stackable /stackable +COPY --chown=stackable:stackable nifi/licenses /licenses +COPY --chown=stackable:stackable nifi/python /stackable/python + RUN curl -L https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-bin.tar.gz | tar -xzC . && \ ln -s /stackable/nifi-${PRODUCT} /stackable/nifi diff --git a/nifi/python/requirements.txt b/nifi/python/requirements.txt deleted file mode 100644 index 788eb166..00000000 --- a/nifi/python/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -nipyapi==0.19.1 \ No newline at end of file From b1836d3ecd29432259112bd53760e4070f92a34e Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 13 Dec 2022 12:46:42 +0100 Subject: [PATCH 12/14] Reduce Trino image size --- trino/Dockerfile | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/trino/Dockerfile b/trino/Dockerfile index e06a6c34..14f55764 100644 --- a/trino/Dockerfile +++ b/trino/Dockerfile @@ -18,22 +18,19 @@ LABEL name="Trino" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install tar gzip zip python3 openssl && \ - microdnf install shadow-utils && \ - microdnf clean all - -COPY trino/stackable /stackable -COPY trino/licenses /licenses + microdnf install tar gzip zip python3 openssl shadow-utils && \ + microdnf clean all && \ + alternatives --set python /usr/bin/python3 RUN groupadd -r stackable --gid=1000 && \ - useradd -r -g stackable --uid=1000 stackable && \ - chown -R stackable:stackable /stackable - -RUN alternatives --set python /usr/bin/python3 + useradd -r -g stackable --uid=1000 stackable USER stackable WORKDIR /stackable +COPY --chown=stackable:stackable trino/stackable /stackable +COPY --chown=stackable:stackable trino/licenses /licenses + RUN curl -L https://repo.stackable.tech/repository/packages/trino-server/trino-server-${PRODUCT}.tar.gz | tar -xzC . && \ ln -s /stackable/trino-server-${PRODUCT} /stackable/trino-server @@ -41,8 +38,7 @@ RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_promet -o /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar && \ chmod -x /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar -WORKDIR /stackable/trino-server/plugin -RUN curl -L https://repo.stackable.tech/repository/packages/trino-opa-authorizer/trino-opa-authorizer-${PRODUCT}-${OPA_AUTHORIZER}.tar.gz | tar -xzC . +RUN curl -L https://repo.stackable.tech/repository/packages/trino-opa-authorizer/trino-opa-authorizer-${PRODUCT}-${OPA_AUTHORIZER}.tar.gz | tar -xzC /stackable/trino-server/plugin # === # Mitigation for CVE-2021-44228 (Log4Shell) From b480506346e77e110459d0d1791bb30d1e17406b Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 13 Dec 2022 12:51:58 +0100 Subject: [PATCH 13/14] Reduce Kafka image size --- kafka/Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index ea60a19a..d68a7756 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -29,20 +29,18 @@ LABEL name="Apache Kafka" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install tar gzip zip && \ - microdnf install shadow-utils openssl && \ + microdnf install tar gzip zip shadow-utils openssl && \ microdnf clean all -COPY kafka/stackable /stackable -COPY kafka/licenses /licenses - RUN groupadd -r stackable --gid=1000 && \ - useradd -r -g stackable --uid=1000 stackable && \ - chown -R stackable:stackable /stackable + useradd -r -g stackable --uid=1000 stackable USER stackable WORKDIR /stackable +COPY --chown=stackable:stackable kafka/stackable /stackable +COPY --chown=stackable:stackable kafka/licenses /licenses + RUN curl -L https://repo.stackable.tech/repository/packages/kafka/kafka_${SCALA}-${PRODUCT}.tgz | tar -xzC . && \ ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka From 2e1cdfaff7d8d83d0608dd1b43233fe0cfb19526 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 13 Dec 2022 12:55:07 +0100 Subject: [PATCH 14/14] Reduce Druid image size --- druid/Dockerfile | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index b64690c7..dae9a257 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -18,21 +18,18 @@ LABEL name="Apache Druid" \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN microdnf update && \ - microdnf install findutils openssl && \ - microdnf install tar gzip zip && \ - microdnf install shadow-utils && \ + microdnf install findutils openssl tar gzip zip shadow-utils && \ microdnf clean all -COPY druid/stackable /stackable -COPY druid/licenses /licenses - RUN groupadd -r stackable --gid=1000 && \ - useradd -r -g stackable --uid=1000 stackable && \ - chown -R stackable:stackable /stackable + useradd -r -g stackable --uid=1000 stackable USER stackable WORKDIR /stackable +COPY --chown=stackable:stackable druid/stackable /stackable +COPY --chown=stackable:stackable druid/licenses /licenses + RUN curl -L https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-bin.tar.gz | tar -xzC . && \ ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid && \ # Force to overwrite the existing 'run-druid'