Fix unstable cluster reliability tests

CHANGELOG.md

@@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file.
 - Add an IT to check that the agent erases its wazuh-agent.state file ([#4716](https://github.com/wazuh/wazuh-qa/pull/4716)) \- (Core)
 - Fix manager_agent system tests environment ([#4808](https://github.com/wazuh/wazuh-qa/pull/4808)) \- (Framework)
 - Fixed agent_simulator response for active-response configuration commands. ([#4895](https://github.com/wazuh/wazuh-qa/pull/4895)) \- (Framework)
+- Fixed stability in cluster reliability tests. ([#5448](https://github.com/wazuh/wazuh-qa/pull/5448)) \- (Framework)
 
 ## [4.8.0] - TBD
 

tests/reliability/test_cluster/test_cluster_logs/test_cluster_connection/test_cluster_connection.py

@@ -7,11 +7,29 @@
 from glob import glob
 from mmap import mmap, ACCESS_READ
 from os.path import join
+from datetime import datetime, timedelta
+from dateutil import parser
+
+DATETIME_FORMAT = '%Y/%m/%d %H:%M'
+SIGTERM_PATTERN = rb'SIGNAL \[\(15\)-\(SIGTERM\)\]'
 
 disconnected_nodes = []
 node_name = re.compile(r'.*/(master|worker_[\d]+)/logs/cluster.log')
 
 
+def get_master_mmap(artifacts_path):
+    """Read the master cluster log and retunr a mmap with the content.
+
+    Args:
+        artifacts_path (str): Path where folders with cluster information can be found.
+
+    Returns:
+        mmap: with the master logs.
+    """
+    with open(join(artifacts_path, 'master', 'logs', 'cluster.log')) as master_log:
+        return mmap(master_log.fileno(), 0, access=ACCESS_READ)
+
+
 def test_cluster_connection(artifacts_path):
     """Verify that no worker disconnects from the master once they are connected.
 
@@ -29,6 +47,8 @@ def test_cluster_connection(artifacts_path):
     if len(cluster_log_files) == 0:
         pytest.fail(f'No files found inside {artifacts_path}.')
 
+    master_mmap = get_master_mmap(artifacts_path=artifacts_path)
+
     for log_file in cluster_log_files:
         with open(log_file) as f:
             s = mmap(f.fileno(), 0, access=ACCESS_READ)
@@ -39,9 +59,27 @@ def test_cluster_connection(artifacts_path):
                             f'{node_name.search(log_file)[1]}')
 
             # Search if there are any connection attempts after the message found above.
-            if re.search(rb'^.*Could not connect to master. Trying.*$|^.*Successfully connected to master.*$',
-                         s[conn.end():], flags=re.MULTILINE):
-                disconnected_nodes.append(node_name.search(log_file)[1])
+            finds = re.search(
+                rb'^.*Could not connect to master. Trying.*$|^.*Successfully connected to master.*$',
+                s[conn.end():],
+                flags=re.MULTILINE
+            )
+            if finds:
+                # Search for SIGTERM in the worker log
+                end_log_timestamp = re.search(rb'(\d{4}\/\d{2}\/\d{2} \d{2}\:\d{2})', finds.group()).group()
+                start_datetime = parser.parse(end_log_timestamp.decode()) - timedelta(minutes=1)
+                start_log_timestamp = start_datetime.strftime(DATETIME_FORMAT)
+
+                start_log = re.search(fr'{start_log_timestamp}.*'.encode(), s)
+                worker_sigterm = re.search(SIGTERM_PATTERN, s[start_log.start():finds.start()])
+
+                if not worker_sigterm:
+                    # Search for SIGTERM in the master log
+                    master_start_log = re.search(fr'{start_log_timestamp}.*'.encode(), master_mmap)
+                    master_sigterm = re.search(SIGTERM_PATTERN, master_mmap[master_start_log.start():])
+
+                    if not master_sigterm:
+                        disconnected_nodes.append(node_name.search(log_file)[1])
 
     if disconnected_nodes:
         pytest.fail(f'The following nodes disconnected from master at any point:\n- ' + '\n- '.join(disconnected_nodes))

tests/reliability/test_cluster/test_cluster_logs/test_cluster_worker_logs_order/data/Integrity_sync.yml

@@ -20,15 +20,9 @@
   tag: 'Updating local files: Start.*'
 - log_id: log13
   parent: log12
-  tag: 'Received [0-9]* missing files to update from master.'
+  tag: 'Updating local files: End.'
 - log_id: log14
   parent: log13
-  tag: 'Received [0-9]* shared files to update from master.*'
-- log_id: log15
-  parent: log14
-  tag: 'Updating local files: End.'
-- log_id: log16
-  parent: log15
   tag: 'Finished in .*'
 
 # No sync needed.

tests/reliability/test_cluster/test_cluster_logs/test_cluster_worker_logs_order/test_cluster_worker_logs_order.py

@@ -40,10 +40,9 @@ def dict_to_tree(dict_tree):
         'node': 'root'
     } for filename in os.listdir(test_data_path)
 }
+order_restarter = re.compile(r'.* The master closed the connection')
 
 
-@pytest.mark.xfail(reason="known cluster log issue due to cluster logging refactor "
-                          "to be worked in https://github.com/wazuh/wazuh/issues/20162")
 def test_check_logs_order_workers(artifacts_path):
     """Check that cluster logs appear in the expected order.
 
@@ -65,7 +64,14 @@ def test_check_logs_order_workers(artifacts_path):
 
         with open(log_file) as file:
             for line in file.readlines():
+
+                if order_restarter.search(line):
+                    for log_order in logs_order.values():
+                        log_order['node'] = 'root'
+                    continue
+
                 result = worker_logs_format.search(line)
+
                 if result:
                     if result.group(1) in logs_order and result.group(1) not in failed_tasks:
                         tree_info = logs_order[result.group(1)]