Merge remote-tracking branch 'origin/main' into logstash-fix-1500

README.md

@@ -59,7 +59,7 @@ exact versions are defined under `KUBERNETES_VERSIONS` in
 ### Helm Versions
 
 While we are checking backward compatibility, the charts are only tested with
-Helm version mentioned in [helm-tester Dockerfile][] (currently 3.6.2).
+Helm version mentioned in [helm-tester Dockerfile][] (currently 3.8.0).
 
 
 ## ECK

elasticsearch/README.md

@@ -174,6 +174,7 @@ support multiple versions with minimal changes.
 | `service.loadBalancerSourceRanges` | The IP ranges that are allowed to access                                                                                                                                                                                                                                                                          | `[]`                                             |
 | `service.nodePort`                 | Custom [nodePort][] port that can be set if you are using `service.type: nodePort`                                                                                                                                                                                                                                | `""`                                             |
 | `service.transportPortName`        | The name of the transport port within the service                                                                                                                                                                                                                                                                 | `transport`                                      |
+| `service.publishNotReadyAddresses` | Consider that all endpoints are considered "ready" even if the Pods themselves are not                                                                                                                                                                                                                            | `false`                                          |
 | `service.type`                     | Elasticsearch [Service Types][]                                                                                                                                                                                                                                                                                   | `ClusterIP`                                      |
 | `sysctlInitContainer`              | Allows you to disable the `sysctlInitContainer` if you are setting [sysctl vm.max_map_count][] with another method                                                                                                                                                                                                | `enabled: true`                                  |
 | `sysctlVmMaxMapCount`              | Sets the [sysctl vm.max_map_count][] needed for Elasticsearch                                                                                                                                                                                                                                                     | `262144`                                         |

elasticsearch/templates/_helpers.tpl

@@ -28,7 +28,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- end -}}
 
 {{/*
-Generate certificates 
+Generate certificates
 */}}
 {{- define "elasticsearch.gen-certs" -}}
 {{- $altNames := list ( printf "%s.%s" (include "elasticsearch.name" .) .Release.Namespace ) ( printf "%s.%s.svc" (include "elasticsearch.name" .) .Release.Namespace ) -}}
@@ -81,3 +81,10 @@ ca.crt: {{ $ca.Cert | toString | b64enc }}
   {{- end -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Use the fullname if the serviceAccount value is not set
+*/}}
+{{- define "elasticsearch.serviceAccount" -}}
+{{- .Values.rbac.serviceAccountName | default (include "elasticsearch.uname" .) -}}
+{{- end -}}

elasticsearch/templates/poddisruptionbudget.yaml

@@ -1,6 +1,9 @@
 {{- if .Values.maxUnavailable }}
----
+{{- if .Capabilities.APIVersions.Has "policy/v1" -}}
+apiVersion: policy/v1
+{{- else}}
 apiVersion: policy/v1beta1
+{{- end }}
 kind: PodDisruptionBudget
 metadata:
   name: "{{ template "elasticsearch.uname" . }}-pdb"

elasticsearch/templates/podsecuritypolicy.yaml

@@ -1,6 +1,10 @@
 {{- if .Values.podSecurityPolicy.create -}}
 {{- $fullName := include "elasticsearch.uname" . -}}
+{{- if .Capabilities.APIVersions.Has "policy/v1" -}}
+apiVersion: policy/v1
+{{- else}}
 apiVersion: policy/v1beta1
+{{- end }}
 kind: PodSecurityPolicy
 metadata:
   name: {{ default $fullName .Values.podSecurityPolicy.name | quote }}

elasticsearch/templates/rolebinding.yaml

@@ -11,11 +11,7 @@ metadata:
     app: {{ $fullName | quote }}
 subjects:
   - kind: ServiceAccount
-    {{- if eq .Values.rbac.serviceAccountName "" }}
-    name: {{ $fullName | quote }}
-    {{- else }}
-    name: {{ .Values.rbac.serviceAccountName | quote }}
-    {{- end }}
+    name: "{{ template "elasticsearch.serviceAccount" . }}"
     namespace: {{ .Release.Namespace | quote }}
 roleRef:
   kind: Role

elasticsearch/templates/service.yaml

@@ -24,6 +24,7 @@ spec:
     release: {{ .Release.Name | quote }}
     chart: "{{ .Chart.Name }}"
     app: "{{ template "elasticsearch.uname" . }}"
+  publishNotReadyAddresses: {{ .Values.service.publishNotReadyAddresses }}
   ports:
   - name: {{ .Values.service.httpPortName | default "http" }}
     protocol: TCP

elasticsearch/templates/serviceaccount.yaml

@@ -3,11 +3,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  {{- if eq .Values.rbac.serviceAccountName "" }}
-  name: {{ $fullName | quote }}
-  {{- else }}
-  name: {{ .Values.rbac.serviceAccountName | quote }}
-  {{- end }}
+  name: "{{ template "elasticsearch.serviceAccount" . }}"
   annotations:
     {{- with .Values.rbac.serviceAccountAnnotations }}
     {{- toYaml . | nindent 4 }}

elasticsearch/templates/statefulset.yaml

@@ -69,10 +69,8 @@ spec:
         {{- if .Values.fsGroup }}
         fsGroup: {{ .Values.fsGroup }} # Deprecated value, please use .Values.podSecurityContext.fsGroup
         {{- end }}
-      {{- if .Values.rbac.create }}
-      serviceAccountName: "{{ template "elasticsearch.uname" . }}"
-      {{- else if not (eq .Values.rbac.serviceAccountName "") }}
-      serviceAccountName: {{ .Values.rbac.serviceAccountName | quote }}
+      {{- if or .Values.rbac.create .Values.rbac.serviceAccountName }}
+      serviceAccountName: "{{ template "elasticsearch.serviceAccount" . }}"
       {{- end }}
       automountServiceAccountToken: {{ .Values.rbac.automountToken }}
       {{- with .Values.tolerations }}

elasticsearch/tests/elasticsearch_test.py

@@ -118,6 +118,7 @@ def test_defaults():
     assert s["metadata"]["name"] == uname
     assert s["metadata"]["annotations"] == {}
     assert s["spec"]["type"] == "ClusterIP"
+    assert s["spec"]["publishNotReadyAddresses"] == False
     assert len(s["spec"]["ports"]) == 2
     assert s["spec"]["ports"][0] == {"name": "http", "port": 9200, "protocol": "TCP"}
     assert s["spec"]["ports"][1] == {
@@ -863,6 +864,17 @@ def test_disabling_non_headless_service():
     assert uname not in r["service"]
 
 
+def test_enabling_service_publishNotReadyAddresses():
+    config = """
+    service:
+      publishNotReadyAddresses: true
+    """
+
+    r = helm_template(config)
+
+    assert r["service"][uname]["spec"]["publishNotReadyAddresses"] == True
+
+
 def test_adding_a_nodePort():
     config = ""
 

elasticsearch/values.yaml

@@ -196,6 +196,9 @@ service:
   labels: {}
   labelsHeadless: {}
   type: ClusterIP
+  # Consider that all endpoints are considered "ready" even if the Pods themselves are not
+  # https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec
+  publishNotReadyAddresses: false
   nodePort: ""
   annotations: {}
   httpPortName: http

helpers/helm-tester/Dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.7
 
-ENV HELM_VERSION=3.6.2
+ENV HELM_VERSION=3.8.0
 
 RUN wget --no-verbose https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
     tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \

helpers/terraform/Dockerfile

@@ -3,7 +3,7 @@ FROM centos:7
 ENV VAULT_VERSION 0.9.3
 ENV TERRAFORM_VERSION=0.11.7
 ENV KUBECTL_VERSION=1.19.9
-ENV HELM_VERSION=3.6.2
+ENV HELM_VERSION=3.8.0
 ENV DOCKER_VERSION=18.09.7
 ENV JQ_VERSION=1.6
 

logstash/README.md

@@ -105,12 +105,12 @@ useful for the [http input plugin][], for instance.
 | `antiAffinityTopologyKey` | The [anti-affinity][] topology key]. By default this will prevent multiple Logstash nodes from running on the same Kubernetes node                                                                                                   | `kubernetes.io/hostname`              |
 | `antiAffinity`            | Setting this to hard enforces the [anti-affinity][] rules. If it is set to soft it will be done "best effort". Other values will be ignored                                                                                          | `hard`                                |
 | `envFrom`                 | Templatable string to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container                                                                                          | `[]`                                  |
-| `extraContainers`         | Templatable string of additional containers to be passed to the `tpl` function                                                                                                                                                       | `""`                                  |
+| `extraContainers`         | Templatable string of additional containers to be passed to the `tpl` function                                                                                                                                                       | `[]`                                  |
 | `extraEnvs`               | Extra [environment variables][] which will be appended to the `env:` definition for the container                                                                                                                                    | `[]`                                  |
-| `extraInitContainers`     | Templatable string of additional `initContainers` to be passed to the `tpl` function                                                                                                                                                 | `""`                                  |
+| `extraInitContainers`     | Templatable string of additional `initContainers` to be passed to the `tpl` function                                                                                                                                                 | `[]`                                  |
 | `extraPorts`              | An array of extra ports to open on the pod                                                                                                                                                                                           | `[]`                                  |
-| `extraVolumeMounts`       | Templatable string of additional `volumeMounts` to be passed to the `tpl` function                                                                                                                                                   | `""`                                  |
-| `extraVolumes`            | Templatable string of additional `volumes` to be passed to the `tpl` function                                                                                                                                                        | `""`                                  |
+| `extraVolumeMounts`       | Templatable string of additional `volumeMounts` to be passed to the `tpl` function                                                                                                                                                   | `[]`                                  |
+| `extraVolumes`            | Templatable string of additional `volumes` to be passed to the `tpl` function                                                                                                                                                        | `[]`                                  |
 | `fullnameOverride`        | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` "                                                                                            | `""`                                  |
 | `hostAliases`             | Configurable [hostAliases][]                                                                                                                                                                                                         | `[]`                                  |
 | `httpPort`                | The http port that Kubernetes will use for the healthchecks and the service                                                                                                                                                          | `9600`                                |

logstash/templates/_helpers.tpl

@@ -18,3 +18,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Use the fullname if the serviceAccount value is not set
+*/}}
+{{- define "logstash.serviceAccount" -}}
+{{- .Values.rbac.serviceAccountName | default (include "logstash.fullname" .) -}}
+{{- end -}}

logstash/templates/poddisruptionbudget.yaml

@@ -1,6 +1,9 @@
----
 {{- if .Values.maxUnavailable }}
+{{- if .Capabilities.APIVersions.Has "policy/v1" -}}
+apiVersion: policy/v1
+{{- else}}
 apiVersion: policy/v1beta1
+{{- end }}
 kind: PodDisruptionBudget
 metadata:
   name: "{{ template "logstash.fullname" . }}-pdb"

logstash/templates/podsecuritypolicy.yaml

@@ -1,6 +1,10 @@
 {{- if .Values.podSecurityPolicy.create -}}
 {{- $fullName := include "logstash.fullname" . -}}
+{{- if .Capabilities.APIVersions.Has "policy/v1" -}}
+apiVersion: policy/v1
+{{- else}}
 apiVersion: policy/v1beta1
+{{- end }}
 kind: PodSecurityPolicy
 metadata:
   name: {{ default $fullName .Values.podSecurityPolicy.name | quote }}

logstash/templates/rolebinding.yaml

@@ -11,11 +11,7 @@ metadata:
     release: {{ .Release.Name | quote }}
 subjects:
   - kind: ServiceAccount
-    {{- if eq .Values.rbac.serviceAccountName "" }}
-    name: {{ $fullName | quote }}
-    {{- else }}
-    name: {{ .Values.rbac.serviceAccountName | quote }}
-    {{- end }}
+    name: "{{ template "logstash.serviceAccount" . }}"
     namespace: {{ .Release.Namespace | quote }}
 roleRef:
   kind: Role

logstash/templates/serviceaccount.yaml

@@ -3,11 +3,7 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  {{- if eq .Values.rbac.serviceAccountName "" }}
-  name: {{ $fullName | quote }}
-  {{- else }}
-  name: {{ .Values.rbac.serviceAccountName | quote }}
-  {{- end }}
+  name: "{{ template "logstash.serviceAccount" . }}"
   annotations:
     {{- with .Values.rbac.serviceAccountAnnotations }}
     {{- toYaml . | nindent 4 }}

logstash/templates/statefulset.yaml

@@ -67,10 +67,8 @@ spec:
       {{- end }}
       securityContext:
 {{ toYaml .Values.podSecurityContext | indent 8 }}
-      {{- if .Values.rbac.create }}
-      serviceAccountName: "{{ template "logstash.fullname" . }}"
-      {{- else if not (eq .Values.rbac.serviceAccountName "") }}
-      serviceAccountName: {{ .Values.rbac.serviceAccountName | quote }}
+      {{- if or .Values.rbac.create .Values.rbac.serviceAccountName }}
+      serviceAccountName: "{{ template "logstash.serviceAccount" . }}"
       {{- end }}
       {{- with .Values.tolerations }}
       tolerations:
@@ -140,7 +138,11 @@ spec:
             name: {{ template "logstash.fullname" . }}-pattern
         {{- end }}
       {{- if .Values.extraVolumes }}
+      {{- if eq "string" (printf "%T" .Values.extraVolumes) }}
 {{ tpl .Values.extraVolumes . | indent 8 }}
+      {{- else }}
+{{ toYaml .Values.extraVolumes | indent 8 }}
+      {{- end }}
       {{- end }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -151,7 +153,11 @@ spec:
       {{- end }}
       {{- if .Values.extraInitContainers }}
       initContainers:
+      {{- if eq "string" (printf "%T" .Values.extraInitContainers) }}
 {{ tpl .Values.extraInitContainers . | indent 6 }}
+      {{- else }}
+{{ toYaml .Values.extraInitContainers | indent 6 }}      
+      {{- end }}
       {{- end }}
       containers:
       - name: "{{ template "logstash.name" . }}"
@@ -212,12 +218,20 @@ spec:
           {{- end -}}
           {{- end -}}
         {{- if .Values.extraVolumeMounts }}
+        {{- if eq "string" (printf "%T" .Values.extraVolumeMounts) }}
 {{ tpl .Values.extraVolumeMounts . | indent 10 }}
+        {{- else }}
+{{ toYaml .Values.extraVolumeMounts | indent 10 }}
+        {{- end }}
         {{- end }}
 {{- if .Values.lifecycle }}
         lifecycle:
 {{ toYaml .Values.lifecycle | indent 10 }}
 {{- end }}
       {{- if .Values.extraContainers }}
+      {{- if eq "string" (printf "%T" .Values.extraContainers) }}
 {{ tpl .Values.extraContainers . | indent 6 }}
+      {{- else }}
+{{ toYaml .Values.extraContainers | indent 6 }}
+      {{- end }}      
       {{- end }}

logstash/tests/logstash_test.py

@@ -178,6 +178,29 @@ def test_adding_a_extra_volume_with_volume_mount():
     } in extraVolumeMounts
 
 
+def test_adding_a_extra_volume_with_volume_mount_as_yaml():
+    config = """
+extraVolumes:
+  - name: extras
+    emptyDir: {}
+extraVolumeMounts:
+  - name: extras
+    mountPath: /usr/share/extras
+    readOnly: true
+"""
+    r = helm_template(config)
+    extraVolume = r["statefulset"][name]["spec"]["template"]["spec"]["volumes"]
+    assert {"name": "extras", "emptyDir": {}} in extraVolume
+    extraVolumeMounts = r["statefulset"][name]["spec"]["template"]["spec"][
+        "containers"
+    ][0]["volumeMounts"]
+    assert {
+        "name": "extras",
+        "mountPath": "/usr/share/extras",
+        "readOnly": True,
+    } in extraVolumeMounts
+
+
 def test_adding_a_extra_container():
     config = """
 extraContainers: |
@@ -194,6 +217,22 @@ def test_adding_a_extra_container():
     } in extraContainer
 
 
+def test_adding_a_extra_container_as_yaml():
+    config = """
+extraContainers:
+  - name: do-something
+    image: busybox
+    command: ['do', 'something']
+"""
+    r = helm_template(config)
+    extraContainer = r["statefulset"][name]["spec"]["template"]["spec"]["containers"]
+    assert {
+        "name": "do-something",
+        "image": "busybox",
+        "command": ["do", "something"],
+    } in extraContainer
+
+
 def test_adding_a_extra_port():
     config = """
 extraPorts:
@@ -225,6 +264,24 @@ def test_adding_a_extra_init_container():
     } in extraInitContainer
 
 
+def test_adding_a_extra_init_container_as_yaml():
+    config = """
+extraInitContainers:
+  - name: do-something
+    image: busybox
+    command: ['do', 'something']
+"""
+    r = helm_template(config)
+    extraInitContainer = r["statefulset"][name]["spec"]["template"]["spec"][
+        "initContainers"
+    ]
+    assert {
+        "name": "do-something",
+        "image": "busybox",
+        "command": ["do", "something"],
+    } in extraInitContainer
+
+
 def test_adding_persistence():
     config = """
 persistence: