From becca8de135f41e94156e8040cf9a0d5c4800553 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 24 Sep 2020 17:34:09 +0200 Subject: [PATCH 1/5] fix: mark flaky test --- auditbeat/module/file_integrity/eventreader_test.go | 2 ++ filebeat/tests/system/test_autodiscover.py | 5 +++++ filebeat/tests/system/test_reload_inputs.py | 3 +++ libbeat/outputs/elasticsearch/client_integration_test.go | 4 ++++ x-pack/auditbeat/tests/system/test_metricsets.py | 3 +++ 5 files changed, 17 insertions(+) diff --git a/auditbeat/module/file_integrity/eventreader_test.go b/auditbeat/module/file_integrity/eventreader_test.go index 6d9b02f7867..8ebad6518fa 100644 --- a/auditbeat/module/file_integrity/eventreader_test.go +++ b/auditbeat/module/file_integrity/eventreader_test.go @@ -41,6 +41,7 @@ func init() { const ErrorSharingViolation syscall.Errno = 32 func TestEventReader(t *testing.T) { + t.Skip("Flaky test: about 1/10 of bulds fails") // Make dir to monitor. dir, err := ioutil.TempDir("", "audit") if err != nil { @@ -240,6 +241,7 @@ func TestEventReader(t *testing.T) { } func TestRaces(t *testing.T) { + t.Skip("Flaky test: about 1/20 of bulds fails") const ( fileMode os.FileMode = 0640 N = 100 diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index 62dd7916437..98d2d331c38 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -1,6 +1,7 @@ import docker import filebeat import os +import pytest import unittest from beat.beat import INTEGRATION_TESTS @@ -11,6 +12,8 @@ class TestAutodiscover(filebeat.BaseTest): """ Test filebeat autodiscover """ + # 1/20 build fails + @pytest.mark.flaky(reruns=1, reruns_delay=10) @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", "integration test not available on 2.x") @@ -42,6 +45,8 @@ def test_docker(self): self.wait_until(lambda: self.log_contains('Stopping runner: input')) proc.check_kill_and_wait() + # 16/20 build fails + @pytest.mark.flaky(reruns=16, reruns_delay=10) @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", "integration test not available on 2.x") diff --git a/filebeat/tests/system/test_reload_inputs.py b/filebeat/tests/system/test_reload_inputs.py index 4de554fe2c6..6aaeaa639d7 100644 --- a/filebeat/tests/system/test_reload_inputs.py +++ b/filebeat/tests/system/test_reload_inputs.py @@ -1,4 +1,5 @@ import os +import pytest import time from filebeat import BaseTest @@ -249,6 +250,8 @@ def test_load_configs(self): assert output[0]["message"] == first_line assert output[1]["message"] == second_line + # 1/20 build fails + @pytest.mark.flaky(reruns=1, reruns_delay=10) def test_reload_same_config(self): """ Test reload same config with same file but different config. Makes sure reloading also works on conflicts. diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index e243cb7d1e4..976de8d7ceb 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -54,7 +54,11 @@ func TestClientPublishEvent(t *testing.T) { testPublishEvent(t, index, cfg) } +// this test is broken 21 fails of 21 builds is not flaky at all +// https://github.com/elastic/beats/issues/21295 +// https://github.com/elastic/beats/pull/21296 func TestClientPublishEventKerberosAware(t *testing.T) { + t.Skip("Flaky test: https://github.com/elastic/beats/issues/21295") err := setupRoleMapping(t, eslegtest.GetEsKerberosHost()) if err != nil { t.Fatal(err) diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index 26bddcef785..115e0ad5ba2 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -1,6 +1,7 @@ import jinja2 import os import platform +import pytest import sys import time import unittest @@ -42,6 +43,8 @@ def test_metricset_login(self): # Metricset is beta and that generates a warning, TODO: remove later self.check_metricset("system", "login", COMMON_FIELDS + fields, config, warnings_allowed=True) + # 1/20 build fails + @pytest.mark.flaky(reruns=1, reruns_delay=10) @unittest.skipIf(sys.platform == "win32", "Not implemented for Windows") @unittest.skipIf(sys.platform.startswith('linux') and not (os.path.isdir("/var/lib/dpkg") or os.path.isdir("/var/lib/rpm")), "Only implemented for dpkg and rpm") From 7864a40ede50b77c5d762fe5e27915b52c1a01c3 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 24 Sep 2020 18:01:35 +0200 Subject: [PATCH 2/5] Apply suggestions from code review --- auditbeat/module/file_integrity/eventreader_test.go | 4 ++-- filebeat/tests/system/test_autodiscover.py | 4 ++-- filebeat/tests/system/test_reload_inputs.py | 2 +- x-pack/auditbeat/tests/system/test_metricsets.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/auditbeat/module/file_integrity/eventreader_test.go b/auditbeat/module/file_integrity/eventreader_test.go index 8ebad6518fa..a00367f350a 100644 --- a/auditbeat/module/file_integrity/eventreader_test.go +++ b/auditbeat/module/file_integrity/eventreader_test.go @@ -41,7 +41,7 @@ func init() { const ErrorSharingViolation syscall.Errno = 32 func TestEventReader(t *testing.T) { - t.Skip("Flaky test: about 1/10 of bulds fails") + t.Skip("Flaky test: about 1/10 of bulds fails https://github.com/elastic/beats/issues/21302") // Make dir to monitor. dir, err := ioutil.TempDir("", "audit") if err != nil { @@ -241,7 +241,7 @@ func TestEventReader(t *testing.T) { } func TestRaces(t *testing.T) { - t.Skip("Flaky test: about 1/20 of bulds fails") + t.Skip("Flaky test: about 1/20 of bulds fails https://github.com/elastic/beats/issues/21303") const ( fileMode os.FileMode = 0640 N = 100 diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index 98d2d331c38..42a45268c6e 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -12,7 +12,7 @@ class TestAutodiscover(filebeat.BaseTest): """ Test filebeat autodiscover """ - # 1/20 build fails + # 1/20 build fails https://github.com/elastic/beats/issues/21305 @pytest.mark.flaky(reruns=1, reruns_delay=10) @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", @@ -45,7 +45,7 @@ def test_docker(self): self.wait_until(lambda: self.log_contains('Stopping runner: input')) proc.check_kill_and_wait() - # 16/20 build fails + # 16/20 build fails https://github.com/elastic/beats/issues/21306 @pytest.mark.flaky(reruns=16, reruns_delay=10) @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", diff --git a/filebeat/tests/system/test_reload_inputs.py b/filebeat/tests/system/test_reload_inputs.py index 6aaeaa639d7..4c5fa1abfba 100644 --- a/filebeat/tests/system/test_reload_inputs.py +++ b/filebeat/tests/system/test_reload_inputs.py @@ -250,7 +250,7 @@ def test_load_configs(self): assert output[0]["message"] == first_line assert output[1]["message"] == second_line - # 1/20 build fails + # 1/20 build fails https://github.com/elastic/beats/issues/21307 @pytest.mark.flaky(reruns=1, reruns_delay=10) def test_reload_same_config(self): """ diff --git a/x-pack/auditbeat/tests/system/test_metricsets.py b/x-pack/auditbeat/tests/system/test_metricsets.py index 115e0ad5ba2..d60c1b82ef8 100644 --- a/x-pack/auditbeat/tests/system/test_metricsets.py +++ b/x-pack/auditbeat/tests/system/test_metricsets.py @@ -43,7 +43,7 @@ def test_metricset_login(self): # Metricset is beta and that generates a warning, TODO: remove later self.check_metricset("system", "login", COMMON_FIELDS + fields, config, warnings_allowed=True) - # 1/20 build fails + # 1/20 build fails https://github.com/elastic/beats/issues/21308 @pytest.mark.flaky(reruns=1, reruns_delay=10) @unittest.skipIf(sys.platform == "win32", "Not implemented for Windows") @unittest.skipIf(sys.platform.startswith('linux') and not (os.path.isdir("/var/lib/dpkg") or os.path.isdir("/var/lib/rpm")), From b32bc8e7cdbb6efa5ccdb7856bf333afd303d199 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Tue, 29 Sep 2020 13:15:36 +0200 Subject: [PATCH 3/5] Update libbeat/outputs/elasticsearch/client_integration_test.go --- libbeat/outputs/elasticsearch/client_integration_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 976de8d7ceb..e243cb7d1e4 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -54,11 +54,7 @@ func TestClientPublishEvent(t *testing.T) { testPublishEvent(t, index, cfg) } -// this test is broken 21 fails of 21 builds is not flaky at all -// https://github.com/elastic/beats/issues/21295 -// https://github.com/elastic/beats/pull/21296 func TestClientPublishEventKerberosAware(t *testing.T) { - t.Skip("Flaky test: https://github.com/elastic/beats/issues/21295") err := setupRoleMapping(t, eslegtest.GetEsKerberosHost()) if err != nil { t.Fatal(err) From eb093d86c64dc0fdd8d9b8b49e8bbd93d2eb1fa1 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Tue, 29 Sep 2020 13:17:53 +0200 Subject: [PATCH 4/5] Apply suggestions from code review --- filebeat/tests/system/test_autodiscover.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index 42a45268c6e..3d4300ecc76 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -12,8 +12,6 @@ class TestAutodiscover(filebeat.BaseTest): """ Test filebeat autodiscover """ - # 1/20 build fails https://github.com/elastic/beats/issues/21305 - @pytest.mark.flaky(reruns=1, reruns_delay=10) @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", "integration test not available on 2.x") @@ -45,8 +43,6 @@ def test_docker(self): self.wait_until(lambda: self.log_contains('Stopping runner: input')) proc.check_kill_and_wait() - # 16/20 build fails https://github.com/elastic/beats/issues/21306 - @pytest.mark.flaky(reruns=16, reruns_delay=10) @unittest.skipIf(not INTEGRATION_TESTS or os.getenv("TESTING_ENVIRONMENT") == "2x", "integration test not available on 2.x") From ad2a4fbee2172adb025d2e48961760db5585dd55 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Tue, 29 Sep 2020 13:19:08 +0200 Subject: [PATCH 5/5] Apply suggestions from code review --- filebeat/tests/system/test_autodiscover.py | 1 - 1 file changed, 1 deletion(-) diff --git a/filebeat/tests/system/test_autodiscover.py b/filebeat/tests/system/test_autodiscover.py index 3d4300ecc76..62dd7916437 100644 --- a/filebeat/tests/system/test_autodiscover.py +++ b/filebeat/tests/system/test_autodiscover.py @@ -1,7 +1,6 @@ import docker import filebeat import os -import pytest import unittest from beat.beat import INTEGRATION_TESTS