🌱 Switch probe tests to helper func (#3782)

* simplify test helper to verify finding outcomes Signed-off-by: Spencer Schrock <sschrock@google.com> * switch existing callers to helper func Signed-off-by: Spencer Schrock <sschrock@google.com> * remove TODO comments Signed-off-by: Spencer Schrock <sschrock@google.com> * fixup doc string Signed-off-by: Spencer Schrock <sschrock@google.com> --------- Signed-off-by: Spencer Schrock <sschrock@google.com>
ossf · Jan 10, 2024 · c59e93b · c59e93b
1 parent 1917fc8
commit c59e93b
Show file tree

Hide file tree

Showing 50 changed files with 105 additions and 514 deletions.
diff --git a/probes/blocksDeleteOnBranches/impl_test.go b/probes/blocksDeleteOnBranches/impl_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 )
 
 func Test_Run(t *testing.T) {
@@ -170,16 +171,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/blocksForcePushOnBranches/impl_test.go b/probes/blocksForcePushOnBranches/impl_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 )
 
 func Test_Run(t *testing.T) {
@@ -170,16 +171,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/branchProtectionAppliesToAdmins/impl_test.go b/probes/branchProtectionAppliesToAdmins/impl_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 )
 
 func Test_Run(t *testing.T) {
@@ -170,16 +171,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/contributorsFromOrgOrCompany/impl_test.go b/probes/contributorsFromOrgOrCompany/impl_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 )
 
 type User struct {
@@ -144,7 +145,6 @@ func Test_Run(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			// TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function.
 			findings, s, err := Run(tt.raw)
 			if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) {
 				t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors()))
@@ -156,16 +156,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/dismissesStaleReviews/impl_test.go b/probes/dismissesStaleReviews/impl_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/clients"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 )
 
 func Test_Run(t *testing.T) {
@@ -188,16 +189,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/freeOfAnyBinaryArtifacts/impl_test.go b/probes/freeOfAnyBinaryArtifacts/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 )
 
 func Test_Run(t *testing.T) {
@@ -143,16 +144,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/freeOfUnverifiedBinaryArtifacts/impl_test.go b/probes/freeOfUnverifiedBinaryArtifacts/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 )
 
 func Test_Run(t *testing.T) {
@@ -108,16 +109,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/fuzzedWithCLibFuzzer/impl_test.go b/probes/fuzzedWithCLibFuzzer/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 	"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
 )
 
@@ -118,7 +119,6 @@ func Test_Run(t *testing.T) {
 		tt := tt // Re-initializing variable so it is not changed while executing the closure below
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-			// TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function.
 			findings, s, err := Run(tt.raw)
 			if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) {
 				t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors()))
@@ -129,16 +129,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/fuzzedWithClusterFuzzLite/impl_test.go b/probes/fuzzedWithClusterFuzzLite/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 	"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
 )
 
@@ -129,16 +130,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/fuzzedWithCppLibFuzzer/impl_test.go b/probes/fuzzedWithCppLibFuzzer/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 	"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
 )
 
@@ -118,7 +119,6 @@ func Test_Run(t *testing.T) {
 		tt := tt // Re-initializing variable so it is not changed while executing the closure below
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-			// TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function.
 			findings, s, err := Run(tt.raw)
 			if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) {
 				t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors()))
@@ -129,16 +129,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/fuzzedWithGoNative/impl_test.go b/probes/fuzzedWithGoNative/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 	"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
 )
 
@@ -129,16 +130,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go b/probes/fuzzedWithJavaJazzerFuzzer/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 	"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
 )
 
@@ -118,7 +119,6 @@ func Test_Run(t *testing.T) {
 		tt := tt // Re-initializing variable so it is not changed while executing the closure below
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
-			// TODO(#https://github.com/ossf/scorecard/issues/3472) Use common validation function.
 			findings, s, err := Run(tt.raw)
 			if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) {
 				t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors()))
@@ -129,16 +129,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/fuzzedWithOSSFuzz/impl_test.go b/probes/fuzzedWithOSSFuzz/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 	"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
 )
 
@@ -129,16 +130,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }
diff --git a/probes/fuzzedWithPropertyBasedHaskell/impl_test.go b/probes/fuzzedWithPropertyBasedHaskell/impl_test.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/internal/utils/test"
 	"github.com/ossf/scorecard/v4/probes/internal/utils/uerror"
 )
 
@@ -129,16 +130,7 @@ func Test_Run(t *testing.T) {
 			if diff := cmp.Diff(Probe, s); diff != "" {
 				t.Errorf("mismatch (-want +got):\n%s", diff)
 			}
-			if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" {
-				t.Errorf("mismatch (-want +got):\n%s", diff)
-			}
-			for i := range tt.outcomes {
-				outcome := &tt.outcomes[i]
-				f := &findings[i]
-				if diff := cmp.Diff(*outcome, f.Outcome); diff != "" {
-					t.Errorf("mismatch (-want +got):\n%s", diff)
-				}
-			}
+			test.AssertOutcomes(t, findings, tt.outcomes)
 		})
 	}
 }