Merge pull request #1002 from wazuh/dev-stabilize-integration-tests

QA Integration tests estabilization - Epic implementation

deps/wazuh_testing/wazuh_testing/data/keepalives.txt

@@ -1,36 +1,36 @@
 mojave
-#!-Darwin |snaow-imac.home |18.5.0 |Darwin Kernel Version 18.5.0: Mon Mar 11 20:40:32 PDT 2019; root:xnu-4903.251.3~3/RELEASE_X86_64 |x86_64 [Mac OS X|darwin: 10.14.4 (Mojave)] - Wazuh v3.11.0 / ab73af41699f13fdd81903b5f23d8d00
+#!-Darwin |snaow-imac.home |18.5.0 |Darwin Kernel Version 18.5.0: Mon Mar 11 20:40:32 PDT 2019; root:xnu-4903.251.3~3/RELEASE_X86_64 |x86_64 [Mac OS X|darwin: 10.14.4 (Mojave)] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":172.16.5.5
 ubuntu12.04
-#!-Linux |agent-ubuntu12 |3.2.0-126-virtual |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 12.04.5 LTS, Precise Pangolin] - Wazuh v3.11.3 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-ubuntu12 |3.2.0-126-virtual |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 12.04.5 LTS, Precise Pangolin] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":172.16.5.6
 ubuntu14.04
-#!-Linux |agent-ubuntu14 |3.13.0-170-generic |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 14.04.6 LTS, Trusty Tahr] - Wazuh v3.11.3 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-ubuntu14 |3.13.0-170-generic |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 14.04.6 LTS, Trusty Tahr] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":172.16.5.7
 ubuntu16.04
-#!-Linux |agent-ubuntu16 |4.4.0-173-generic |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 16.04.6 LTS (Xenial Xerus)] - Wazuh v3.11.3 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-ubuntu16 |4.4.0-173-generic |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 16.04.6 LTS (Xenial Xerus)] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":172.16.5.8
 ubuntu18.04
-#!-Linux |agent-ubuntu18 |4.15.0-76-generic |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 18.04.4 LTS (Bionic Beaver)] - Wazuh v3.11.3 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-ubuntu18 |4.15.0-76-generic |#220-Ubuntu SMP Thu May 9 12:40:49 UTC 2019 |x86_64 [Ubuntu|ubuntu: 18.04.4 LTS (Bionic Beaver)] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":172.16.5.9
 debian7
-#!-Linux |agent-debian7 |3.16.0-9-amd64 |#1 SMP Debian 3.16.68-1 (2019-05-22) |x86_64 [Debian GNU/Linux|debian: 7 (wheezy)] - Wazuh v3.11.3 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-debian7 |3.16.0-9-amd64 |#1 SMP Debian 3.16.68-1 (2019-05-22) |x86_64 [Debian GNU/Linux|debian: 7 (wheezy)] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":10.0.2.15
 debian8
-#!-Linux |agent-debian8 |3.16.0-9-amd64 |#1 SMP Debian 3.16.68-1 (2019-05-22) |x86_64 [Debian GNU/Linux|debian: 8 (jessie)] - Wazuh v4.1.0 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-debian8 |3.16.0-9-amd64 |#1 SMP Debian 3.16.68-1 (2019-05-22) |x86_64 [Debian GNU/Linux|debian: 8 (jessie)] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":10.0.2.15
 debian9
-#!-Linux |agent-debian9 |4.9.0-12-amd64 |#1 SMP Debian 4.9.210-1 (2020-01-20) |x86_64 [Debian GNU/Linux|debian: 9 (stretch)] - Wazuh v3.11.3 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-debian9 |4.9.0-12-amd64 |#1 SMP Debian 4.9.210-1 (2020-01-20) |x86_64 [Debian GNU/Linux|debian: 9 (stretch)] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":10.0.2.15
 debian10
-#!-Linux |agent-debian10 |4.9.0-12-amd64 |#1 SMP Debian 4.9.210-1 (2020-01-20) |x86_64 [Debian GNU/Linux|debian: 10 (buster)] - Wazuh v3.11.3 / ab73af41699f13fdd81903b5f23d8d00
+#!-Linux |agent-debian10 |4.9.0-12-amd64 |#1 SMP Debian 4.9.210-1 (2020-01-20) |x86_64 [Debian GNU/Linux|debian: 10 (buster)] - Wazuh <VERSION> / ab73af41699f13fdd81903b5f23d8d00
 d6e3ac3e75ca0319af3e7c262776f331 merged.mg
 #"_agent_ip":10.0.2.15

deps/wazuh_testing/wazuh_testing/tools/agent_simulator.py

@@ -39,13 +39,16 @@
 class Agent:
     def __init__(self, manager_address, cypher="aes", os=None,
                  inventory_sample=None, rootcheck_sample= None,
-                 id=None, name=None, key=None, version="3.12",
+                 id=None, name=None, key=None, version="v3.12.0",
                  fim_eps=None, fim_integrity_eps=None,
                  authd_password=None):
         self.id = id
         self.name = name
         self.key = key
-        self.version = version
+        if version is not None:
+            self.long_version = version
+            ver_split = version.replace("v","").split(".")
+            self.short_version = f"{'.'.join(ver_split[:2])}"
         self.cypher = cypher
         self.os = os
         self.fim_eps = 1000 if fim_eps is None else fim_eps
@@ -97,14 +100,12 @@ def set_os(self):
     # Set variables related to wpk simulated responses
     def set_wpk_variables(self, sha=None, upgrade_exec_result=None,
                           upgrade_notification=False, upgrade_script_result=0,
-                          stage_disconnect=None, version=None):
+                          stage_disconnect=None):
         self.sha_key = sha
         self.upgrade_exec_result = upgrade_exec_result
         self.send_upgrade_notification = upgrade_notification
         self.upgrade_script_result = upgrade_script_result
         self.stage_disconnect = stage_disconnect
-        if version:
-            self.version = version
 
     # Set agent name
     def set_name(self):
@@ -286,7 +287,7 @@ def processCommand(self, sender, message_list):
                             self.stage_disconnect == 'clear_upgrade_result':
                 self.stop_receive = 1
             else:
-                if float(self.version) < 4.1 or command == 'lock_restart':
+                if self.short_version < "4.1" or command == 'lock_restart':
                     sender.sendEvent(self.createEvent(f'#!-req {message_list[1]} ok '))
                 else:
                     sender.sendEvent(self.createEvent(f'#!-req {message_list[1]} '
@@ -297,7 +298,7 @@ def processCommand(self, sender, message_list):
                 if command == 'sha1' and self.stage_disconnect == 'sha1':
                     self.stop_receive = 1
                 else:
-                    if float(self.version) < 4.1:
+                    if self.short_version < "4.1":
                         sender.sendEvent(self.createEvent(f'#!-req {message_list[1]} '
                                                           f'ok {self.sha_key}'))
                     else:
@@ -310,7 +311,7 @@ def processCommand(self, sender, message_list):
                 if command == 'upgrade' and self.stage_disconnect == 'upgrade':
                     self.stop_receive = 1
                 else:
-                    if float(self.version) < 4.1:
+                    if self.short_version < "4.1":
                         sender.sendEvent(self.createEvent(
                                          f'#!-req {message_list[1]} ok '
                                          f'{self.upgrade_exec_result}'))
@@ -353,6 +354,7 @@ def createKeepAlive(self):
                         line = fp.readline()
                     break
                 line = fp.readline()
+        msg = msg.replace("<VERSION>",self.long_version)
         self.keep_alive_msg = self.createEvent(msg)
 
     def initializeModules(self):
@@ -361,10 +363,10 @@ def initializeModules(self):
         if self.modules["rootcheck"]["status"] == "enabled":
             self.rootcheck = Rootcheck(self.rootcheck_sample)
         if self.modules["fim"]["status"] == "enabled":
-            self.fim = GeneratorFIM(self.id, self.name, self.version)
+            self.fim = GeneratorFIM(self.id, self.name, self.short_version)
         if self.modules["fim_integrity"]["status"] == "enabled":
             self.fim_integrity = GeneratorIntegrityFIM(self.id, self.name,
-                                                       self.version)
+                                                       self.short_version)
 
 
 class Inventory:
@@ -621,11 +623,11 @@ def getAttributes(self):
         return attributes
 
     def formatMessage(self, message):
-        if self.agent_version == "3.12":
+        if self.agent_version >= "3.12":
             return '{0}:[{1}] ({2}) any->syscheck:{3}' \
                     .format(self.SYSCHECK_MQ, self.agent_id,
                             self.agent_name, message)
-        if self.agent_version == "3.11":
+        else:
             # If first time generating. Send control message to simulate
             # end of FIM baseline.
             if self.baseline_completed == 0:
@@ -636,7 +638,7 @@ def formatMessage(self, message):
                                         message)
 
     def generateMessage(self):
-        if self.agent_version == "3.12":
+        if self.agent_version >= "3.12":
             if self.event_type == "added":
                 timestamp = int(time())
                 self.generateAttributes()
@@ -667,7 +669,7 @@ def generateMessage(self):
 
             message = json.dumps({"type": "event", "data": data})
 
-        if self.agent_version == "3.11":
+        else:
             self.generateAttributes()
             message = '{0}:{1}:{2}:{3}:{4}:{5}:{6}:{7}:{8}:{9} {10}'.format(
                    self._size, self._mode, self._uid, self._gid, self._md5,
@@ -879,7 +881,7 @@ def stop_rec(self):
 
 
 def create_agents(agents_number, manager_address, cypher, fim_eps=None,
-                  authd_password=None, os=None):
+                  authd_password=None, os=None, version=None):
     global agent_count
     # Read client.keys and create virtual agents
     agents = []
@@ -888,11 +890,14 @@ def create_agents(agents_number, manager_address, cypher, fim_eps=None,
             agent_os = os[agent]
         else:
             agent_os = None
+
+        agent_version = version[agent] if version is not None else None
+
         if authd_password is not None:
             agents.append(Agent(manager_address, cypher, fim_eps=fim_eps,
-                                authd_password=authd_password, os=agent_os))
+                                authd_password=authd_password, os=agent_os, version=agent_version))
         else:
             agents.append(Agent(manager_address, cypher, fim_eps=fim_eps,
-                                os=agent_os))
+                                os=agent_os, version=agent_version))
         agent_count = agent_count + 1
     return agents

tests/integration/README.md

@@ -141,13 +141,19 @@ Finally, copy your `wazuh-qa` repository within your testing environment and you
 
 Our newest integration tests are located in `wazuh-qa/tests/integration/`. They are organized by capabilities:
 
+- _test_agentd_
 - _test_analysisd_
+- _test_api_
+- _test_authd_
 - _test_cluster_
 - _test_fim_
 - _test_gcloud_
-- _test_mitre_
+- _test_logtest_
+- _test_rids_
+- _test_rootcheck_
+- _test_vulnerability_detector_
 - _test_wazuh_db_
-- _test_sca_
+- _test_wpk_
 
 Every group will have the following structure:
 
@@ -367,6 +373,7 @@ python3 -m pytest [options] [file_or_dir] [file_or_dir] [...]
 is not met before the given time lapse. Some tests make use of this value and other has other fixed timeout that cannot be
 modified.
 - `--fim_mode`: Specify the mode of execution of the FIM tests. (ex. --fim_mode="scheduled"). To run the test in realtime and whodata the option must be specified twice: --fim_mode="realtime" --fim_mode="whodata". If the option is not specified, the test will run using scheduled, whodata and realtime.
+- `--wpk_version`: Specify the WPK package version used to upgrade on WPK tests. (ex. --wpk_version=v4.2.0). Note: This field is required to execute any WPK test and the WPK package must be previously created in the [repository](packages-dev.wazuh.com/trash/wpk/).
 
 _Use `-h` to see the rest or check its [documentation](https://docs.pytest.org/en/latest/usage.html)._
 

tests/integration/conftest.py

@@ -173,6 +173,14 @@ def pytest_addoption(parser):
         type=str,
         help="run tests using a specific FIM mode"
     )
+    parser.addoption(
+        "--wpk_version",
+        action="append",
+        metavar="wpk_version",
+        default=None,
+        type=str,
+        help="run tests using a specific WPK package version"
+    )
 
 
 def pytest_configure(config):
@@ -217,6 +225,8 @@ def pytest_configure(config):
         mode = ["scheduled", "whodata", "realtime"]
     global_parameters.fim_mode = mode
 
+    # Set WPK package version
+    global_parameters.wpk_version = config.getoption("--wpk_version")  
 
 def pytest_html_results_table_header(cells):
     cells.insert(4, html.th('Tier', class_='sortable tier', col='tier'))

tests/integration/test_api/test_config/test_behind_proxy_server/test_behind_proxy_server.py

@@ -35,6 +35,7 @@ def get_configuration(request):
 
 # Tests
 
+@pytest.mark.xfail(reason='To be deprecated in https://github.com/wazuh/wazuh/issues/7006')
 @pytest.mark.parametrize('tags_to_apply', [
     {'bps_enabled'},
     {'bps_disabled'},

tests/integration/test_authd/test_authd_agents_ctx.py

@@ -65,7 +65,6 @@ def get_configuration(request):
 def clean_agents_ctx():  
     clean_keys()
     clean_groups()
-    clean_agentinfo()
     clean_rids()
     clean_agents_timestamp()
     clean_diff()
@@ -104,15 +103,6 @@ def clean_diff():
         except Exception as e:
             print('Failed to delete %s. Reason: %s' % (diff_path, e))
 
-def clean_agentinfo():
-    agentinfo_folder = os.path.join(WAZUH_PATH, 'queue', 'agent-info')
-    for filename in os.listdir(agentinfo_folder):
-        file_path = os.path.join(agentinfo_folder, filename)
-        try:
-            os.unlink(file_path)
-        except Exception as e:
-            print('Failed to delete %s. Reason: %s' % (file_path, e))
-
 def clean_rids():
     rids_folder = os.path.join(WAZUH_PATH, 'queue', 'rids')
     for filename in os.listdir(rids_folder):
@@ -258,7 +248,7 @@ def register_agent_local_server(Name, Group=None, IP=None):
 # Tests
 def duplicate_ip_agent_delete_test(server): 
     """Register a first agent, then register an agent with duplicated IP.
-        Check that client.keys, agent-info, agent-groups, agent-timestamp and agent diff were updated correctly
+        Check that client.keys, agent-groups, agent-timestamp and agent diff were updated correctly
 
     Parameters
     ----------
@@ -300,7 +290,7 @@ def duplicate_ip_agent_delete_test(server):
 
 def duplicate_name_agent_delete_test(server): 
     """Register a first agent, then register an agent with duplicated Name.
-        Check that client.keys, agent-info, agent-groups, agent-timestamp and agent diff were updated correctly
+        Check that client.keys, agent-groups, agent-timestamp and agent diff were updated correctly
 
     Parameters
     ----------

tests/integration/test_cluster/test_key_polling/test_key_polling_master.py

@@ -60,6 +60,7 @@ def get_configuration(request):
 
 # Tests
 
+@pytest.mark.skip(reason='Development in progress: https://github.com/wazuh/wazuh/issues/4387')
 @pytest.mark.parametrize('cmd, counter, payload, expected', [
     (b'run_keypoll', 1, b'{"message": "id:001"}', "id:001"),
     (b'run_keypoll', 2, b'{"message": "ip:124.0.0.1"}', "ip:124.0.0.1")
@@ -85,7 +86,6 @@ def test_key_polling_master(cmd, counter, payload, expected, configure_environme
     expected : str
         Expected message in krequest socket
     """
-    pytest.xfail("Development in progress: https://github.com/wazuh/wazuh/issues/4387")
     # Build message and send it to the master
     message = cluster_msg_build(cmd=cmd, counter=counter, payload=payload, encrypt=True)
     receiver_sockets[0].send(message)

tests/integration/test_cluster/test_key_polling/test_key_polling_worker.py

@@ -55,6 +55,7 @@ def get_configuration(request):
 # Tests
 
 
+@pytest.mark.skip(reason='Development in progress: https://github.com/wazuh/wazuh/issues/4387')
 @pytest.mark.parametrize('cmd, counter, payload', [
     (b'run_keypoll', 1, b'{"message": "id:001"}'),
     (b'run_keypoll', 2, b'{"message": "ip:124.0.0.1"}')
@@ -78,7 +79,6 @@ def test_key_polling_worker(cmd, counter, payload, configure_environment, config
     payload : bytes
         Cluster message payload data
     """
-    pytest.xfail("Development in progress: https://github.com/wazuh/wazuh/issues/4387")
     # Build message to send to c-internal.sock in the worker and send it
     message = cluster_msg_build(cmd=cmd, counter=counter, payload=payload, encrypt=False)
     receiver_sockets[0].send(message)

tests/integration/test_gcloud/test_configuration/test_remote_configuration.py

@@ -76,7 +76,6 @@ def get_remote_configuration(component_name, config):
     socket_path = os.path.join(WAZUH_PATH, 'queue', 'ossec')
     dest_socket = os.path.join(socket_path, component_name)
     command = f"getconfig {config}"
-    host_type = 'agent' if 'agent' in WAZUH_SERVICE else 'server'
 
     # Socket connection
     s = SocketController(dest_socket)
@@ -94,13 +93,9 @@ def get_remote_configuration(component_name, config):
     try:
         if rec_msg_ok.startswith('ok'):
             remote_configuration = json.loads(rec_msg)
-            if host_type == 'server':
-                remote_configuration_gcp = remote_configuration['wmodules'][6]['gcp-pubsub']
-            else:
-                if sys.platform == 'darwin':
-                    remote_configuration_gcp = remote_configuration['wmodules'][3]['gcp-pubsub']
-                else:
-                    remote_configuration_gcp = remote_configuration['wmodules'][5]['gcp-pubsub']
+            for element in remote_configuration['wmodules']:
+                if 'gcp-pubsub' in element:
+                    remote_configuration_gcp = element['gcp-pubsub']
         else:
             s.close()
             raise ValueError(rec_msg_ok)

tests/integration/test_gcloud/test_functioning/test_rules.py

@@ -84,11 +84,9 @@ def test_rules(get_configuration, configure_environment,
     rules_id = []
     file_ind = 0
 
-    for number in range(65004, 65037):
-        rules_id.append(number)
-
-    for number in range(65039, 65045):
-        rules_id.append(number)
+    rules_id = [id for id in range(65005, 65011)]
+    rules_id += [id for id in range(65012, 65039)]
+    rules_id += [id for id in range(65041, 65047)]
 
     events_file = open(file_path, 'r')
     for line in events_file:

tests/integration/test_wpk/conftest.py

@@ -9,5 +9,5 @@ def configure_agents(request, get_configuration):
     SERVER_ADDRESS = getattr(request.module, 'SERVER_ADDRESS')
     CRYPTO = getattr(request.module, 'CRYPTO')
 
-    agents = create_agents(agents_number, SERVER_ADDRESS, CRYPTO,  os=metadata['agents_os'])
+    agents = create_agents(agents_number, SERVER_ADDRESS, CRYPTO,  os=metadata['agents_os'], version=metadata['agents_version'])
     setattr(request.module, 'agents', agents)