From 8c98263631df0e120365adb7601095daf881cf16 Mon Sep 17 00:00:00 2001 From: Chris Lovett Date: Fri, 15 Oct 2021 16:01:41 -0700 Subject: [PATCH 1/3] add some handy warnings about non-static coyote test methods. --- .../Test/SystematicTesting/TestMethodInfo.cs | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/Source/Test/SystematicTesting/TestMethodInfo.cs b/Source/Test/SystematicTesting/TestMethodInfo.cs index 569c4c268..354fa24ec 100644 --- a/Source/Test/SystematicTesting/TestMethodInfo.cs +++ b/Source/Test/SystematicTesting/TestMethodInfo.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; +using System.Text; using System.Threading.Tasks; using Microsoft.Coyote.Actors; using Microsoft.Coyote.IO; @@ -156,8 +157,30 @@ private static (Delegate testMethod, string testName) GetTestMethod(Assembly ass } else if (testMethods.Count is 0) { - throw new InvalidOperationException("Cannot detect a Coyote test method declared with the " + - $"'[{typeof(TestAttribute).FullName}]' attribute."); + // see if user forgot to make it static! + flags = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; + testMethods = FindTestMethodsWithAttribute(typeof(TestAttribute), flags, assembly); + StringBuilder sb = new StringBuilder(); + if (testMethods.Count > 0) + { + foreach (var method in testMethods) + { + string wrong = "non-static"; + if (!method.IsPublic) + { + wrong += " and non-public"; + } + + sb.AppendLine($"Ignoring Coyote test method '{method.DeclaringType.Name + "." + method.Name}' because it is {wrong}"); + } + + throw new InvalidOperationException($"The following Coyote test methods cannot be used :\n{sb}"); + } + else + { + throw new InvalidOperationException($"{sb}Cannot detect a static Coyote test method declared with the " + + $"'[{typeof(TestAttribute).FullName}]' attribute."); + } } MethodInfo testMethod = testMethods[0]; @@ -235,6 +258,13 @@ private static MethodInfo GetTestSetupMethod(Assembly assembly, Type attribute) if (testMethods.Count is 0) { + flags = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; + testMethods = FindTestMethodsWithAttribute(attribute, flags, assembly); + if (testMethods.Count > 0) + { + throw new InvalidOperationException($"Test setup methods must be static and public: '{testMethods[0].Name}'."); + } + return null; } else if (testMethods.Count > 1) @@ -294,8 +324,9 @@ private static List FindTestMethodsWithAttribute(Type attribute, Bin try { + var name = attribute.FullName; // so we can test an assembly built against a different version of Coyote.exe. testMethods = assembly.GetTypes().SelectMany(t => t.GetMethods(bindingFlags)). - Where(m => m.GetCustomAttributes(attribute, false).Length > 0).ToList(); + Where(m => m.GetCustomAttributes(attribute, false).Any()).ToList(); } catch (ReflectionTypeLoadException ex) { From 5dd510f73687b1fbba525f35a20a3cbceacc4981 Mon Sep 17 00:00:00 2001 From: Chris Lovett Date: Mon, 18 Oct 2021 10:44:46 -0700 Subject: [PATCH 2/3] remove unused code. --- Source/Test/SystematicTesting/TestMethodInfo.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Test/SystematicTesting/TestMethodInfo.cs b/Source/Test/SystematicTesting/TestMethodInfo.cs index 354fa24ec..5b0a4c015 100644 --- a/Source/Test/SystematicTesting/TestMethodInfo.cs +++ b/Source/Test/SystematicTesting/TestMethodInfo.cs @@ -324,7 +324,6 @@ private static List FindTestMethodsWithAttribute(Type attribute, Bin try { - var name = attribute.FullName; // so we can test an assembly built against a different version of Coyote.exe. testMethods = assembly.GetTypes().SelectMany(t => t.GetMethods(bindingFlags)). Where(m => m.GetCustomAttributes(attribute, false).Any()).ToList(); } From 779f36b1b7de495b8921eb208f352d1c71d3c2c2 Mon Sep 17 00:00:00 2001 From: Chris Lovett Date: Mon, 18 Oct 2021 10:58:05 -0700 Subject: [PATCH 3/3] Improved error messages --- .../Test/SystematicTesting/TestMethodInfo.cs | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/Source/Test/SystematicTesting/TestMethodInfo.cs b/Source/Test/SystematicTesting/TestMethodInfo.cs index 5b0a4c015..23061bd98 100644 --- a/Source/Test/SystematicTesting/TestMethodInfo.cs +++ b/Source/Test/SystematicTesting/TestMethodInfo.cs @@ -158,27 +158,15 @@ private static (Delegate testMethod, string testName) GetTestMethod(Assembly ass else if (testMethods.Count is 0) { // see if user forgot to make it static! - flags = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; + flags = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic; testMethods = FindTestMethodsWithAttribute(typeof(TestAttribute), flags, assembly); - StringBuilder sb = new StringBuilder(); if (testMethods.Count > 0) { - foreach (var method in testMethods) - { - string wrong = "non-static"; - if (!method.IsPublic) - { - wrong += " and non-public"; - } - - sb.AppendLine($"Ignoring Coyote test method '{method.DeclaringType.Name + "." + method.Name}' because it is {wrong}"); - } - - throw new InvalidOperationException($"The following Coyote test methods cannot be used :\n{sb}"); + ReportBadTestMethods(testMethods, "test"); } else { - throw new InvalidOperationException($"{sb}Cannot detect a static Coyote test method declared with the " + + throw new InvalidOperationException($"Cannot detect a public static Coyote test method declared with the " + $"'[{typeof(TestAttribute).FullName}]' attribute."); } } @@ -247,6 +235,29 @@ private static (Delegate testMethod, string testName) GetTestMethod(Assembly ass return (test, $"{testMethod.DeclaringType}.{testMethod.Name}"); } + private static void ReportBadTestMethods(List testMethods, string caption) + { + StringBuilder sb = new StringBuilder(); + + foreach (var method in testMethods) + { + string wrong = null; + if (!method.IsStatic) + { + wrong = "not static"; + } + + if (!method.IsPublic) + { + wrong = string.IsNullOrEmpty(wrong) ? "not public" : wrong + " and not public"; + } + + sb.AppendLine($"Ignoring method '{method.DeclaringType.Name + "." + method.Name}' because it is {wrong}"); + } + + throw new InvalidOperationException($"The following Coyote {caption} methods cannot be used :\n{sb}"); + } + /// /// Returns the test method with the specified attribute. /// Returns null if no such method is found. @@ -258,11 +269,11 @@ private static MethodInfo GetTestSetupMethod(Assembly assembly, Type attribute) if (testMethods.Count is 0) { - flags = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic; + flags = BindingFlags.DeclaredOnly | BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic; testMethods = FindTestMethodsWithAttribute(attribute, flags, assembly); if (testMethods.Count > 0) { - throw new InvalidOperationException($"Test setup methods must be static and public: '{testMethods[0].Name}'."); + ReportBadTestMethods(testMethods, "test setup"); } return null;