Add E2E tests

Signed-off-by: Raghav Kaul <raghavkaul@google.com>

attestor/command/check.go

@@ -35,6 +35,11 @@ func (ep EmptyParameterError) Error() string {
 }
 
 func runCheck() (policy.PolicyResult, error) {
+	return RunCheckWithParams(repoURL, commitSHA, policyPath)
+}
+
+// RunCheckWithParams: Run scorecard check on repo. Export for testability.
+func RunCheckWithParams(repoURL, commitSHA, policyPath string) (policy.PolicyResult, error) {
 	ctx := context.Background()
 	logger := sclog.NewLogger(sclog.DefaultLevel)
 

e2e/attestor_policy_test.go

@@ -0,0 +1,145 @@
+// Copyright 2021 Security Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v2"
+
+	"github.com/ossf/scorecard/v4/attestor/command"
+	"github.com/ossf/scorecard/v4/attestor/policy"
+)
+
+var _ = Describe("E2E TEST PAT: scorecard-attestor policy", func() {
+	Context("E2E TEST:Validating scorecard attestation policy", func() {
+		It("Should attest to repos based on policy", func() {
+			tt := []struct {
+				name     string
+				repoURL  string
+				policy   policy.AttestationPolicy
+				expected policy.PolicyResult
+			}{
+				{
+					name:    "test good repo",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-good",
+					policy: policy.AttestationPolicy{
+						PreventBinaryArtifacts:      true,
+						PreventKnownVulnerabilities: true,
+						PreventUnpinnedDependencies: true,
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test bad repo with policies disabled",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
+					policy: policy.AttestationPolicy{
+						PreventBinaryArtifacts:      false,
+						PreventKnownVulnerabilities: true,
+						PreventUnpinnedDependencies: false,
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test bad repo with ignored binary artifact",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
+					policy: policy.AttestationPolicy{
+						PreventBinaryArtifacts:      true,
+						AllowedBinaryArtifacts:      []string{"test-binary-artifact-*"},
+						PreventKnownVulnerabilities: true,
+						PreventUnpinnedDependencies: false,
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test bad repo with ignored dep by path",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
+					policy: policy.AttestationPolicy{
+						PreventBinaryArtifacts:      false,
+						PreventUnpinnedDependencies: true,
+						AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}},
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test bad repo with ignored dep by name",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
+					policy: policy.AttestationPolicy{
+						PreventBinaryArtifacts:      false,
+						PreventUnpinnedDependencies: true,
+						AllowedUnpinnedDependencies: []policy.Dependency{{PackageName: "static-debian11"}, {PackageName: "golang"}},
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test bad repo with everything ignored",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
+					policy: policy.AttestationPolicy{
+						PreventBinaryArtifacts:      true,
+						AllowedBinaryArtifacts:      []string{"test-binary-artifact-*"},
+						PreventKnownVulnerabilities: true,
+						PreventUnpinnedDependencies: true,
+						AllowedUnpinnedDependencies: []policy.Dependency{{Filepath: "Dockerfile"}},
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test repo with simple code review requirements",
+					repoURL: "https://github.com/ossf/scorecard",
+					policy: policy.AttestationPolicy{
+						EnsureCodeReviewed: true,
+						CodeReviewRequirements: policy.CodeReviewRequirements{
+							MinReviewers: 1,
+						},
+					},
+					expected: policy.Pass,
+				},
+				{
+					name:    "test code reviews required but repo doesn't have code reviews",
+					repoURL: "https://github.com/ossf-tests/scorecard-binauthz-test-bad",
+					policy: policy.AttestationPolicy{
+						PreventBinaryArtifacts:      true,
+						PreventKnownVulnerabilities: true,
+						PreventUnpinnedDependencies: true,
+						EnsureCodeReviewed:          true,
+					},
+					expected: policy.Fail,
+				},
+			}
+
+			for _, tc := range tt {
+				fmt.Printf("attestor_policy_test.go: %s\n", tc.name)
+				f, err := os.CreateTemp("/tmp", strings.ReplaceAll(tc.name, " ", "-"))
+				Expect(err).Should(BeNil())
+				defer os.Remove(f.Name())
+
+				buf, err := yaml.Marshal(tc.policy)
+				Expect(err).Should(BeNil())
+
+				nbytes, err := f.Write(buf)
+				Expect(err).Should(BeNil())
+				Expect(nbytes).Should(BeNumerically(">", 0))
+
+				result, err := command.RunCheckWithParams(tc.repoURL, "HEAD", f.Name())
+				Expect(err).Should(BeNil())
+				Expect(result).Should(BeEquivalentTo(tc.expected))
+			}
+		})
+	})
+})