mongo node syn test alert implementation

scality · Sep 4, 2024 · 225aeff · 225aeff
1 parent 70181fa
commit 225aeff
Showing 1 changed file with 80 additions and 0 deletions.
diff --git a/monitoring/mongodb/alerts.test.yaml b/monitoring/mongodb/alerts.test.yaml
@@ -258,5 +258,85 @@ tests:
             exp_annotations:
               description: MongoDb has low disk space
               summary: MongoDb has low disk space
+
+  - name: MongoDbNodeRecovering
+    interval: 1m
+    input_series:
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-mongos-0", rs_state="3"}
+        values: 3x60
+
+    alert_rule_test:
+      - alertname: MongoDbNodeRecovering
+        eval_time: 1h
+        exp_alerts:
+          - exp_labels:
+              namespace: zenko
+              severity: warning
+              pod: data-db-mongodb-sharded-mongos-0
+              rs_state: 3
+            exp_annotations:
+              description: "The Mongodb instance `data-db-mongodb-sharded-mongos-0` is in the 'RECOVERING' state for over an hour. The instance may not be able to join the replica set if the platform ingests a large number of operations during this time. This alert is expected if the 'Resync a Data Services MongoDB Member' procedure has recently been executed."
+              summary: MongoDB node recovering for too long
+
+  - name: MongoDbInvalidState
+    interval: 1m
+    input_series:
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-mongos-0", rs_state="6"}
+        values: 6x10
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-mongos-0", rs_state="8"}
+        values: 0x10
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-mongos-0", rs_state="10"}
+        values: 0x10
+
+    alert_rule_test:
+      - alertname: MongoDbInvalidState
+        eval_time: 1m
+        exp_alerts:
+          - exp_labels:
+              namespace: zenko
+              severity: critical
+              pod: data-db-mongodb-sharded-mongos-0
+              rs_state: 6
+            exp_annotations:
+              description: "The Mongodb instance `data-db-mongodb-sharded-mongos-0` is in the `6` state. Immediate attention is required."
+              summary: "MongoDB node in an invalid state: 6 (UNKNOWN), 8 (DOWN), 10 (REMOVED)"
+
+  - name: MongoDbNodeStartup2
+    interval: 1m
+    input_series:
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-mongos-0", rs_state="5"}
+        values: 5x60
+
+    alert_rule_test:
+      - alertname: MongoDbNodeStartup2
+        eval_time: 1h
+        exp_alerts:
+          - exp_labels:
+              namespace: zenko
+              severity: warning
+              pod: data-db-mongodb-sharded-mongos-0
+              rs_state: 5
+            exp_annotations:
+              description: "The Mongodb instance `data-db-mongodb-sharded-mongos-0` is in the 'STARTUP2' state for an hour. The instance might be stuck."
+              summary: MongoDB node in STARTUP2 state for too long
+
+
+  - name: MongoDbNodeNotSynced
+    interval: 1m
+    input_series:
+      - series: mongodb_mongod_replset_number_of_members{set="data-db-mongodb-sharded-shard-0", pod="mongodb-1"}
+        values: 1+0x9
+
+    alert_rule_test:
+      - alertname: MongoDbNodeNotSynced
+        eval_time: 1m
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              pod: mongodb-1
+            exp_annotations:
+              description: "The MongoDB instance `mongodb-1` is out of the replica set. It does not longer receive any data and must be added back to the cluster to avoid performance and storage problems."
+              summary: MongoDB node not in replica set
+