Upgrade vector (#465)

* adapt for vector upgrade * patch operator-rs * changelog * bump vector aggregator * bump operator-rs * stackable version * switch tests to sentEventsTotal * new line * add filtered invalid events * attempt to fix linter * adapt kuttl test * align versions
stackabletech · Aug 3, 2023 · b22b6c1 · b22b6c1
1 parent 5a3e9eb
commit b22b6c1
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,12 @@ All notable changes to this project will be documented in this file.
 
 - Default stackableVersion to operator version ([#467]).
 
+### Changed
+
+- `vector` `0.26.0` -> `0.31.0` ([#465]).
+- `operator-rs` `0.44.0` -> `0.45.1` ([#467]).
+
+[#465]: https://github.com/stackabletech/opa-operator/pull/465
 [#467]: https://github.com/stackabletech/opa-operator/pull/467
 
 ## [23.7.0] - 2023-07-14

diff --git a/tests/kuttl-test.yaml.jinja2 b/tests/kuttl-test.yaml.jinja2
@@ -9,3 +9,19 @@ testDirs:
 startKIND: false
 suppress: ["events"]
 parallel: 2
+
+# The timeout (in seconds) is used when namespaces are created or
+# deleted, and, if not overridden, in TestSteps, TestAsserts, and
+# Commands. If not set, the timeout is 30 seconds by default.
+#
+# The deletion of a namespace can take a while until all resources are
+# gracefully shut down. If the timeout is reached in the meantime, even
+# a successful test case is considered a failure.
+#
+# For instance, the termination grace period of the Vector aggregator in
+# the logging tests is set to 60 seconds. If there are logs entries
+# which could not be forwarded yet to the external aggregator defined in
+# the VECTOR_AGGREGATOR environment variable, then the test aggregator
+# uses this period of time by trying to forward the events. In this
+# case, deleting a namespace with several Pods takes about 90 seconds.
+timeout: 120
diff --git a/tests/templates/kuttl/logging/01-install-opa-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-opa-vector-aggregator.yaml
@@ -5,7 +5,7 @@ commands:
   - script: >-
       helm install opa-vector-aggregator vector
       --namespace $NAMESPACE
-      --version 0.19.0
+      --version 0.23.0
       --repo https://helm.vector.dev
       --values opa-vector-aggregator-values.yaml
 ---

diff --git a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2
@@ -20,36 +20,48 @@ customConfig:
       type: vector
       version: "2"
   transforms:
-    automaticLogConfigServerOpa:
+    filteredAutomaticLogConfigServerOpa:
       type: filter
       inputs: [vector]
       condition: >-
         starts_with(string!(.pod), "test-opa-server-automatic-log-config") &&
         .container == "opa"
-    automaticLogConfigServerBundleBuilder:
+    filteredAutomaticLogConfigServerBundleBuilder:
       type: filter
       inputs: [vector]
       condition: >-
         starts_with(string!(.pod), "test-opa-server-automatic-log-config") &&
         .container == "bundle-builder"
-    automaticLogConfigServerVector:
+    filteredAutomaticLogConfigServerVector:
       type: filter
       inputs: [vector]
       condition: >-
         starts_with(string!(.pod), "test-opa-server-automatic-log-config") &&
         .container == "vector"
-    automaticLogConfigServerPrepare:
+    filteredAutomaticLogConfigServerPrepare:
       type: filter
       inputs: [vector]
       condition: >-
         starts_with(string!(.pod), "test-opa-server-automatic-log-config") &&
         .container == "prepare"
+    filteredInvalidEvents:
+      type: filter
+      inputs: [vector]
+      condition: |-
+        .timestamp == from_unix_timestamp!(0) ||
+        is_null(.level) ||
+        is_null(.logger) ||
+        is_null(.message)
   sinks:
     out:
-      inputs: [automaticLogConfig*]
+      inputs: [filtered*]
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
       type: vector
       address: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+      buffer:
+        # Avoid back pressure from VECTOR_AGGREGATOR. The test should
+        # not fail if the aggregator is not available.
+        when_full: drop_newest
 {% else %}
       type: blackhole
 {% endif %}
diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py
@@ -2,7 +2,7 @@
 import requests
 
 
-def check_processed_events():
+def check_sent_events():
     response = requests.post(
         'http://opa-vector-aggregator:8686/graphql',
         json={
@@ -12,8 +12,8 @@ def check_processed_events():
                         nodes {
                             componentId
                             metrics {
-                                processedEventsTotal {
-                                    processedEventsTotal
+                                sentEventsTotal {
+                                    sentEventsTotal
                                 }
                             }
                         }
@@ -30,12 +30,19 @@ def check_processed_events():
 
     transforms = result['data']['transforms']['nodes']
     for transform in transforms:
-        processedEvents = transform['metrics']['processedEventsTotal']['processedEventsTotal']
+        sentEvents = transform['metrics']['sentEventsTotal']
         componentId = transform['componentId']
-        assert processedEvents > 0, \
-            f'No events were processed in "{componentId}".'
+
+        if componentId == 'filteredInvalidEvents':
+            assert sentEvents is None or \
+                   sentEvents['sentEventsTotal'] == 0, \
+                   'Invalid log events were sent.'
+        else:
+            assert sentEvents is not None and \
+                   sentEvents['sentEventsTotal'] > 0, \
+                   f'No events were sent in "{componentId}".'
 
 
 if __name__ == '__main__':
-    check_processed_events()
+    check_sent_events()
     print('Test successful!')