Bump otelcontribcol and helm version

To upgrade to otelcontribcol 0.106.0 we need to mitigate two breaking changes in https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.104.0.

The opencensus receiver depends on 0.0.0.0 so disabling feature gate component.UseLocalHostAsDefaultHost.

With style `$FOO` deprecation, all env vars are changed to match the style. This brings a problem in the otel-agent container start when reconciler-manager and resource-group-controller do not initialize the following env vars:

- `configsync_sync_kind`
- `configsync_sync_name`
- `configsync_sync_namespace`

So splitting the otel-agent config between the reconciler and controllers.

Makefile

@@ -69,14 +69,14 @@ KUSTOMIZE_VERSION := v5.4.2-gke.0
 KUSTOMIZE := $(BIN_DIR)/kustomize
 KUSTOMIZE_STAGING_DIR := $(OUTPUT_DIR)/third_party/kustomize
 
-HELM_VERSION := v3.14.4-gke.2
+HELM_VERSION := v3.15.3-gke.0
 HELM := $(BIN_DIR)/helm
 HELM_STAGING_DIR := $(OUTPUT_DIR)/third_party/helm
 
 GIT_SYNC_VERSION := v4.2.3-gke.5__linux_amd64
 GIT_SYNC_IMAGE_NAME := gcr.io/config-management-release/git-sync:$(GIT_SYNC_VERSION)
 
-OTELCONTRIBCOL_VERSION := v0.102.0-gke.6
+OTELCONTRIBCOL_VERSION := v0.106.0-gke.2
 OTELCONTRIBCOL_IMAGE_NAME := gcr.io/config-management-release/otelcontribcol:$(OTELCONTRIBCOL_VERSION)
 
 # Keep KIND_VERSION in sync with the version defined in go.mod

manifests/base/kustomization.yaml

@@ -22,6 +22,7 @@ resources:
 - ../ns-reconciler-base-cluster-role.yaml
 - ../root-reconciler-base-cluster-role.yaml
 - ../otel-agent-cm.yaml
+- ../otel-agent-reconciler-cm.yaml
 - ../reconciler-manager-service-account.yaml
 - ../reposync-crd.yaml
 - ../rootsync-crd.yaml

manifests/otel-agent-cm.yaml

@@ -32,24 +32,6 @@ data:
         tls:
           insecure: true
     processors:
-      # Attributes processor adds custom configsync metric labels to applicable
-      # metrics to identify the sync object used to configure this deployment.
-      #
-      # Note: configsync.sync.generation is explicitly excluded here, because it
-      # is high cardinality. So we don't want to send it as a label, only as a
-      # resource attribute. That way it's only propagated to Prometheus, and not
-      # Monarch or Cloud Monitoring, which ignore custom resource attributes.
-      attributes:
-        actions:
-          - key: configsync.sync.kind
-            action: upsert
-            value: $CONFIGSYNC_SYNC_KIND
-          - key: configsync.sync.name
-            action: upsert
-            value: $CONFIGSYNC_SYNC_NAME
-          - key: configsync.sync.namespace
-            action: upsert
-            value: $CONFIGSYNC_SYNC_NAMESPACE
       batch:
       # Populate resource attributes from OTEL_RESOURCE_ATTRIBUTES env var and
       # the GCE metadata service, if available.
@@ -62,7 +44,7 @@ data:
       pipelines:
         metrics:
           receivers: [opencensus]
-          processors: [batch, resourcedetection, attributes]
+          processors: [batch, resourcedetection]
           exporters: [opencensus]
       telemetry:
         logs:

manifests/otel-agent-reconciler-cm.yaml

@@ -0,0 +1,69 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: otel-agent-reconciler
+  namespace: config-management-system
+  labels:
+    app: opentelemetry
+    component: otel-agent
+    configmanagement.gke.io/system: "true"
+    configmanagement.gke.io/arch: "csmr"
+data:
+  otel-agent-reconciler-config.yaml: |
+    receivers:
+      opencensus:
+    exporters:
+      opencensus:
+        endpoint: otel-collector.config-management-monitoring:55678
+        tls:
+          insecure: true
+    processors:
+      # Attributes processor adds custom configsync metric labels to applicable
+      # metrics to identify the sync object used to configure this deployment.
+      #
+      # Note: configsync.sync.generation is explicitly excluded here, because it
+      # is high cardinality. So we don't want to send it as a label, only as a
+      # resource attribute. That way it's only propagated to Prometheus, and not
+      # Monarch or Cloud Monitoring, which ignore custom resource attributes.
+      attributes:
+        actions:
+          - key: configsync.sync.kind
+            action: upsert
+            value: ${CONFIGSYNC_SYNC_KIND}
+          - key: configsync.sync.name
+            action: upsert
+            value: ${CONFIGSYNC_SYNC_NAME}
+          - key: configsync.sync.namespace
+            action: upsert
+            value: ${CONFIGSYNC_SYNC_NAMESPACE}
+      batch:
+      # Populate resource attributes from OTEL_RESOURCE_ATTRIBUTES env var and
+      # the GCE metadata service, if available.
+      resourcedetection:
+        detectors: [env, gcp]
+    extensions:
+      health_check:
+    service:
+      extensions: [health_check]
+      pipelines:
+        metrics:
+          receivers: [opencensus]
+          processors: [batch, resourcedetection, attributes]
+          exporters: [opencensus]
+      telemetry:
+        logs:
+          level: "INFO"

manifests/templates/otel-collector.yaml

@@ -101,6 +101,7 @@ spec:
         # The prometheus transformer appends `_ratio` to gauge metrics: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.86.0/pkg/translator/prometheus/normalize_name.go#L149
         # Add the feature gate to enable metric suffix trimming.
         - "--feature-gates=-pkg.translator.prometheus.NormalizeName"
+        - "--feature-gates=-component.UseLocalHostAsDefaultHost"
         resources:
           limits:
             cpu: 1

manifests/templates/reconciler-manager-configmap.yaml

@@ -168,10 +168,11 @@ data:
            command:
            - /otelcontribcol
            args:
-           - "--config=/conf/otel-agent-config.yaml"
+           - "--config=/conf/otel-agent-reconciler-config.yaml"
            # The prometheus transformer appends `_ratio` to gauge metrics: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.86.0/pkg/translator/prometheus/normalize_name.go#L149
            # Add the feature gate to enable metric suffix trimming.
            - "--feature-gates=-pkg.translator.prometheus.NormalizeName"
+           - "--feature-gates=-component.UseLocalHostAsDefaultHost"
            securityContext:
              allowPrivilegeEscalation: false
              readOnlyRootFilesystem: true
@@ -184,7 +185,7 @@ data:
            - containerPort: 8888  # Metrics.
              protocol: TCP
            volumeMounts:
-           - name: otel-agent-config-vol
+           - name: otel-agent-config-reconciler-vol
              mountPath: /conf
            readinessProbe:
              httpGet:
@@ -273,9 +274,9 @@ data:
            secret:
              secretName: git-creds
              defaultMode: 288
-         - name: otel-agent-config-vol
+         - name: otel-agent-config-reconciler-vol
            configMap:
-             name: otel-agent
+             name: otel-agent-reconciler
              defaultMode: 420
          - name: service-account
            emptyDir: {}

manifests/templates/reconciler-manager.yaml

@@ -71,6 +71,7 @@ spec:
         # The prometheus transformer appends `_ratio` to gauge metrics: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.86.0/pkg/translator/prometheus/normalize_name.go#L149
         # Add the feature gate to enable metric suffix trimming.
         - "--feature-gates=-pkg.translator.prometheus.NormalizeName"
+        - "--feature-gates=-component.UseLocalHostAsDefaultHost"
         resources:
           limits:
             cpu: 1

manifests/templates/resourcegroup-manifest.yaml

@@ -231,6 +231,8 @@ spec:
             - ALL
       - args:
         - --config=/conf/otel-agent-config.yaml
+        - --feature-gates=-component.UseLocalHostAsDefaultHost
+        - --feature-gates=-pkg.translator.prometheus.NormalizeName
         command:
         - /otelcontribcol
         env: