Label flaky tests with xfail (#5029)

* Refs #21244: Add xfail tests to BB tests Signed-off-by: Mario Dominguez <mariodominguez@eprosima.com> * Refs #21244: Add xfail tests to communication and performance tests Signed-off-by: Mario Dominguez <mariodominguez@eprosima.com> * Refs #21244: Correct xfail lists Signed-off-by: Mario Dominguez <mariodominguez@eprosima.com> * Refs #21289: Add more flakies Signed-off-by: Mario Dominguez <mariodominguez@eprosima.com> * Refs #21244: Add another round of flakies Signed-off-by: Mario Dominguez <mariodominguez@eprosima.com> --------- Signed-off-by: Mario Dominguez <mariodominguez@eprosima.com>
eProsima · Jul 4, 2024 · 5b39dcb · 5b39dcb
1 parent 1a278d4
commit 5b39dcb
Show file tree

Hide file tree

Showing 13 changed files with 115 additions and 0 deletions.
diff --git a/test/blackbox/XFAIL_DDS_PIM.list b/test/blackbox/XFAIL_DDS_PIM.list
@@ -6,6 +6,7 @@ BlackboxTests_DDS_PIM.LivelinessQos.ShortLiveliness_ManualByTopic_Automatic_Best
 BlackboxTests_DDS_PIM.LivelinessQos.ShortLiveliness_ManualByTopic_Automatic_Reliable.Transport
 BlackboxTests_DDS_PIM.LivelinessQos.ThreeWriters_ThreeReaders.Intraprocess
 BlackboxTests_DDS_PIM.LivelinessQos.ThreeWriters_ThreeReaders.Transport
+BlackboxTests_DDS_PIM.LivelinessQos.ThreeWriters_ThreeReaders.Datasharing
 BlackboxTests_DDS_PIM.LivelinessQos.TwoWriters_OneReader_ManualByParticipant.Intraprocess
 BlackboxTests_DDS_PIM.PersistenceLargeData.PubSubAsReliablePubPersistentWithFrag.Transport
 BlackboxTests_DDS_PIM.ChainingTransportTests.builtin_transports_api_large_data
@@ -14,3 +15,59 @@ BlackboxTests_DDS_PIM.ChainingTransportTests.builtin_transports_env_large_data
 BlackboxTests_DDS_PIM.ChainingTransportTests.builtin_transports_env_large_datav6
 BlackboxTests_DDS_PIM.ChainingTransportTests.builtin_transports_xml_large_data
 BlackboxTests_DDS_PIM.ChainingTransportTests.builtin_transports_xml_large_datav6
+BlackboxTests_DDS_PIM.DeadlineQos.KeyedTopicNoReaderVolatileWriterSetDeadline.Transport
+BlackboxTests_DDS_PIM.DeadlineQos.KeyedTopicNoReaderVolatileWriterSetDeadline.Intraprocess
+BlackboxTests_DDS_PIM.DeadlineQos.KeyedTopicNoReaderVolatileWriterSetDeadline.Datasharing
+BlackboxTests_DDS_PIM.DeadlineQos.KeyedTopicBestEffortReaderVolatileWriterSetDeadline.Intraprocess
+BlackboxTests_DDS_PIM.DeadlineQos.KeyedTopicBestEffortReaderVolatileWriterSetDeadline.Transport
+BlackboxTests_DDS_PIM.TransportTCP.large_data_topology.Transport
+BlackboxTests_DDS_PIM.LivelinessQos.TwoWriters_TwoReaders.Intraprocess
+BlackboxTests_DDS_PIM.DDSDataReader.LivelinessChangedStatusGet.Intraprocess
+BlackboxTests_DDS_PIM.PubSubBasic.ReliableTransientLocalTwoWritersConsecutives.Datasharing
+BlackboxTests_DDS_PIM.PubSubFragments.PubSubAsReliableData300kbSmallFragments.Transport
+BlackboxTests_DDS_PIM.DDSDataWriter.HeartbeatWhileDestruction
+BlackboxTests_DDS_PIM.DDSStatus.IncompatibleQosConditions.Datasharing
+BlackboxTests_DDS_PIM.PubSubBasic.ReliableTransientLocalTwoWritersConsecutives.Datasharing
+BlackboxTests_DDS_PIM.PubSubHistory.PubSubAsReliableKeepAllWithoutKeyAndMaxSamples.Datasharing
+BlackboxTests_DDS_PIM.RealtimeAllocations.AsyncPubSubBestEffortWithLimitedSubscribers.Datasharing
+BlackboxTests_DDS_PIM.Volatile.VolatileKeepAllPubReliableSubNonReliableHelloWorld.Intraprocess
+BlackboxTests_DDS_PIM.Discovery.TwentyParticipantsUnicast.Intraprocess
+BlackboxTests_DDS_PIM.DDSStatus.IncompatibleQosGetters.Intraprocess
+BlackboxTests_DDS_PIM.DDSDataSharing.DataSharingReader_DifferentDomainWriters
+BlackboxTests_DDS_PIM.LivelinessQos.TwoWriters_TwoReaders_ManualByParticipant.Datasharing
+BlackboxTests_DDS_PIM.PubSubFragments.PubSubAsNonReliableVolatileData300kb.Intraprocess
+BlackboxTests_DDS_PIM.PubSubFragments.PubSubAsNonReliableVolatileData300kb.Datasharing
+BlackboxTests_DDS_PIM.PubSubFragments.AsyncPubSubAsNonReliableData300kbSmallFragments.Datasharing
+BlackboxTests_DDS_PIM.PubSubFragments.AsyncPubSubAsReliableTransientLocalData300kb.Transport
+BlackboxTests_DDS_PIM.PubSubHistory.PubSubAsReliableKeepLastReaderSmallDepth.Datasharing
+BlackboxTests_DDS_PIM.Reliability.DisableHeartbeatPiggybackTrue
+BlackboxTests_DDS_PIM.TransportUDP.whitelisting_udp_localhost_multi.Transport
+BlackboxTests_DDS_PIM.Volatile.AsyncVolatileSubBetweenPubs.Intraprocess
+BlackboxTests_DDS_PIM.PubSubFragments.AsyncPubSubAsNonReliableVolatileData300kbSmallFragments.Datasharing
+BlackboxTests_DDS_PIM.RealtimeAllocations.AsyncPubSubBestEffortWithLimitedSubscribers.Intraprocess
+BlackboxTests_DDS_PIM.ChainingTransportTests.builtin_transports_env_udpv6
+BlackboxTests_DDS_PIM.Volatile.VolatileSubBetweenPubs.Transport
+BlackboxTests_DDS_PIM.DeadlineQos.KeyedTopicLongDeadline.Transport
+BlackboxTests_DDS_PIM.Discovery.TwentyParticipantsSeveralEndpointsUnicast.Datasharing
+BlackboxTests_DDS_PIM.LivelinessQos.ShortLiveliness_ManualByTopic_Automatic_BestEffort.Datasharing
+BlackboxTests_DDS_PIM.PubSubFragments.PubSubAsNonReliableVolatileData300kbSmallFragments.Datasharing
+BlackboxTests_DDS_PIM.PubSubFragments.PubSubAsReliableVolatileData300kbSmallFragments.Transport
+BlackboxTests_DDS_PIM.Volatile.AsyncPubSubAsNonReliableVolatileHelloworld.Datasharing
+BlackboxTests_DDS_PIM.DDSContentFilter.BasicTest.Transport
+BlackboxTests_DDS_PIM.Discovery.TwentyParticipantsSeveralEndpointsUnicast.Datasharing
+BlackboxTests_DDS_PIM.LivelinessQos.LongLiveliness_ManualByTopic_Reliable.Datasharing
+BlackboxTests_DDS_PIM.RealtimeAllocations.AsyncPubSubBestEffortWithLimitedSubscribers.Datasharing
+BlackboxTests_DDS_PIM.DDSStatus.sample_lost_waitset_re_dw_be_dr
+BlackboxTests_DDS_PIM.PubSubFragments.PubSubAsReliableVolatileData300kb.Transport
+BlackboxTests_DDS_PIM.PubSubHistory.PubReliableKeepAllSubNonReliable.Intraprocess
+BlackboxTests_DDS_PIM.PubSubHistory.PubSubAsReliableKeepAllWithKeyAndMaxSamplesPerInstance.Transport
+BlackboxTests_DDS_PIM.PubSubHistory.PubSubAsReliableKeepLastWithKeyUnorderedReception.Intraprocess
+BlackboxTests_DDS_PIM.Volatile.AsyncVolatileKeepAllPubReliableSubNonReliableHelloWorld.Intraprocess
+BlackboxTests_DDS_PIM.LivelinessQos.ShortLiveliness_ManualByTopic_Reliable.Intraprocess
+BlackboxTests_DDS_PIM.PubSubBasic.ReliableVolatileTwoWritersConsecutives.Intraprocess
+BlackboxTests_DDS_PIM.PubSubFragments.AsyncPubSubAsReliableData300kbSmallFragments.Datasharing
+BlackboxTests_DDS_PIM.PubSubHistory.PubSubAsReliableMultithreadKeepLast1.Datasharing
+BlackboxTests_DDS_PIM.Volatile.AsyncVolatileSubBetweenTransientPubs.Transport
+BlackboxTests_DDS_PIM.DDSDiscovery.ServersConnectionTCP
+BlackboxTests_DDS_PIM.TransportTCP.TCPDomainHelloWorld_P0_P3_D0_D0.Transport
+BlackboxTests_DDS_PIM.TransportTCP.TCPDomainHelloWorld_P3_P0_D1_D0.Transport
diff --git a/test/blackbox/XFAIL_DDS_PIM_SECURITY.list b/test/blackbox/XFAIL_DDS_PIM_SECURITY.list
@@ -0,0 +1,10 @@
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndCryptoPlugin_reliable_rtps_large_string.Datasharing
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsDisableDiscoveryDisableAccessEncrypt_validation_ok_disable_discovery_enable_access_none.Transport
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsDisableDiscoveryDisableAccessEncrypt_validation_ok_disable_discovery_enable_access_none.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsDisableDiscoveryEnableAccessNone_validation_ok_enable_discovery_enable_access_none.Datasharing
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryDisableAccessNone_validation_ok_disable_discovery_enable_access_none.Transport
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndCryptoPlugin_reliable_rtps_data300kb.Transport
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndCryptoPlugin_reliable_payload_data300kb.Transport
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndCryptoPlugin_besteffort_payload_large_string.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessEncrypt_validation_ok_enable_discovery_disable_access_none.Datasharing
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessNone_validation_ok_disable_discovery_enable_access_none.Datasharing
diff --git a/test/blackbox/XFAIL_RTPS.list b/test/blackbox/XFAIL_RTPS.list
@@ -0,0 +1,2 @@
+BlackboxTests_RTPS.Persistence.RTPSAsNonReliableWithPersistence.Transport
+BlackboxTests_RTPS.RTPS.MultithreadedWriterCreation
diff --git a/test/communication/CMakeLists.txt b/test/communication/CMakeLists.txt
@@ -645,3 +645,10 @@ if(Python3_Interpreter_FOUND)
     endif()
 
 endif()
+
+# Add 'xfail' label to flaky tests
+set(COMMUNICATION_XFAIL_LIST XFAIL_COMMUNICATION)
+
+foreach(COMMUNICATION_XFAIL_TEST ${COMMUNICATION_XFAIL_LIST})
+    add_xfail_label(${CMAKE_CURRENT_SOURCE_DIR}/${COMMUNICATION_XFAIL_TEST}.list)
+endforeach()
diff --git a/test/communication/XFAIL_COMMUNICATION.list b/test/communication/XFAIL_COMMUNICATION.list
@@ -0,0 +1,5 @@
+TwoPublishersCommunicationBestEffort
+SimpleCommunicationReliable
+SHMCommunicationSubscriberDiesWhileProcessingMessage
+AutomaticLivelinessAssertion
+
diff --git a/test/dds/communication/CMakeLists.txt b/test/dds/communication/CMakeLists.txt
@@ -206,3 +206,10 @@ if(Python3_Interpreter_FOUND)
         endif()
     endforeach()
 endif()
+
+# Add 'xfail' label to flaky tests
+set(DDS_COMMUNICATION_XFAIL_LIST XFAIL_DDS_COMMUNICATION)
+
+foreach(DDS_COMMUNICATION_XFAIL_TEST ${DDS_COMMUNICATION_XFAIL_LIST})
+    add_xfail_label(${CMAKE_CURRENT_SOURCE_DIR}/${DDS_COMMUNICATION_XFAIL_TEST}.list)
+endforeach()
diff --git a/test/dds/communication/XFAIL_DDS_COMMUNICATION.list b/test/dds/communication/XFAIL_DDS_COMMUNICATION.list
@@ -0,0 +1,4 @@
+DDSCommunication_zero_copy_pub_communication
+DDSCommunication_zero_copy_communication
+DDSCommunication_mix_zero_copy_communication
+
diff --git a/test/performance/CMakeLists.txt b/test/performance/CMakeLists.txt
@@ -28,3 +28,4 @@ add_subdirectory(throughput)
 if(VIDEO_TESTS)
     add_subdirectory(video)
 endif()
+
diff --git a/test/performance/latency/CMakeLists.txt b/test/performance/latency/CMakeLists.txt
@@ -302,3 +302,10 @@ if(Python3_Interpreter_FOUND)
 
     endforeach(latency_test_name)
 endif()
+
+# Add 'xfail' label to flaky tests
+set(LATENCY_XFAIL_LIST XFAIL_LATENCY)
+
+foreach(LATENCY_XFAIL_TEST ${LATENCY_XFAIL_LIST})
+    add_xfail_label(${CMAKE_CURRENT_SOURCE_DIR}/${LATENCY_XFAIL_TEST}.list)
+endforeach()
diff --git a/test/performance/latency/XFAIL_LATENCY.list b/test/performance/latency/XFAIL_LATENCY.list
@@ -0,0 +1,3 @@
+performance.latency.interprocess_best_effort_shm.security
+performance.latency.interprocess_reliable_shm
+performance.latency.interprocess_best_effort_shm.data_loans_and_sharing
diff --git a/test/performance/throughput/CMakeLists.txt b/test/performance/throughput/CMakeLists.txt
@@ -303,3 +303,10 @@ if(Python3_Interpreter_FOUND)
 
     endforeach(throughput_test_name)
 endif()
+
+# Add 'xfail' label to flaky tests
+set(THROUGHPUT_XFAIL_LIST XFAIL_THROUGHPUT)
+
+foreach(THROUGHPUT_XFAIL_TEST ${THROUGHPUT_XFAIL_LIST})
+    add_xfail_label(${CMAKE_CURRENT_SOURCE_DIR}/${THROUGHPUT_XFAIL_TEST}.list)
+endforeach()
diff --git a/test/performance/throughput/XFAIL_THROUGHPUT.list b/test/performance/throughput/XFAIL_THROUGHPUT.list
@@ -0,0 +1,4 @@
+performance.throughput.interprocess_best_effort_shm.data_loans_and_sharing
+performance.throughput.interprocess_best_effort_udp
+performance.throughput.interprocess_best_effort_udp.data_loans
+performance.throughput.intraprocess_best_effort.data_sharing
diff --git a/test/unittest/transport/XFAIL_TCP4.list b/test/unittest/transport/XFAIL_TCP4.list
@@ -0,0 +1 @@
+TCPv4Tests.reconnect_after_open_port_failure