✨ probe: releases with verified provenance

checks/evaluation/signed_releases.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ossf/scorecard/v5/finding"
 	"github.com/ossf/scorecard/v5/probes/releasesAreSigned"
 	"github.com/ossf/scorecard/v5/probes/releasesHaveProvenance"
+	"github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance"
 )
 
 var errNoReleaseFound = errors.New("no release found")
@@ -55,6 +56,10 @@ func SignedReleases(name string,
 	for i := range findings {
 		f := &findings[i]
 
+		if f.Probe == releasesHaveVerifiedProvenance.Probe {
+			continue
+		}
+
 		// Debug release name
 		if f.Outcome == finding.OutcomeNotApplicable {
 			// Generic summary.
@@ -86,6 +91,10 @@ func SignedReleases(name string,
 	for i := range findings {
 		f := &findings[i]
 
+		if f.Probe == releasesHaveVerifiedProvenance.Probe {
+			continue
+		}
+
 		releaseName := getReleaseName(f)
 		if releaseName == "" {
 			return checker.CreateRuntimeErrorResult(name, errNoReleaseFound)

finding/finding.go

@@ -144,12 +144,24 @@ func NewFalse(efs embed.FS, probeID, text string, loc *Location,
 	return NewWith(efs, probeID, text, loc, OutcomeFalse)
 }
 
+// NewNotApplicable create a finding with a NotApplicable outcome and the desired location.
+func NewNotApplicable(efs embed.FS, probeID, text string, loc *Location,
+) (*Finding, error) {
+	return NewWith(efs, probeID, text, loc, OutcomeNotApplicable)
+}
+
 // NewNotAvailable create a finding with a NotAvailable outcome and the desired location.
 func NewNotAvailable(efs embed.FS, probeID, text string, loc *Location,
 ) (*Finding, error) {
 	return NewWith(efs, probeID, text, loc, OutcomeNotAvailable)
 }
 
+// NewNotSupported create a finding with a NotSupported outcome and the desired location.
+func NewNotSupported(efs embed.FS, probeID, text string, loc *Location,
+) (*Finding, error) {
+	return NewWith(efs, probeID, text, loc, OutcomeNotSupported)
+}
+
 // NewTrue create a true finding with the desired location.
 func NewTrue(efs embed.FS, probeID, text string, loc *Location,
 ) (*Finding, error) {

finding/probe.go

@@ -142,7 +142,7 @@ func validateID(actual, expected string) error {
 
 func validateRemediation(r yamlRemediation) error {
 	if err := validateRemediationOutcomeTrigger(r.OnOutcome); err != nil {
-		return err
+		return fmt.Errorf("remediation: %w", err)
 	}
 	switch r.Effort {
 	case RemediationEffortHigh, RemediationEffortMedium, RemediationEffortLow:

options/flags.go

@@ -193,6 +193,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 	allowedFormats := []string{
 		FormatDefault,
 		FormatJSON,
+		FormatProbe,
 	}
 
 	if o.isSarifEnabled() {

probes/entries.go

@@ -48,6 +48,7 @@ import (
 	"github.com/ossf/scorecard/v5/probes/pinsDependencies"
 	"github.com/ossf/scorecard/v5/probes/releasesAreSigned"
 	"github.com/ossf/scorecard/v5/probes/releasesHaveProvenance"
+	"github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance"
 	"github.com/ossf/scorecard/v5/probes/requiresApproversForPullRequests"
 	"github.com/ossf/scorecard/v5/probes/requiresCodeOwnersReview"
 	"github.com/ossf/scorecard/v5/probes/requiresLastPushApproval"
@@ -168,6 +169,7 @@ var (
 		hasPermissiveLicense.Run,
 		codeReviewOneReviewers.Run,
 		hasBinaryArtifacts.Run,
+		releasesHaveVerifiedProvenance.Run,
 	}
 
 	// Probes which don't use pre-computed raw data but rather collect it themselves.

probes/releasesHaveVerifiedProvenance/def.yml

@@ -0,0 +1,34 @@
+# Copyright 2024 OpenSSF Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+id: releasesHaveVerifiedProvenance
+short: Checks if the project releases with provenance attestations that have been verified
+motivation: >
+  Package provenance attestations provide a greater guarantee of authenticity and integrity than package signatures alone, since the attestation can be performed over a hash of both the package contents and metadata. Developers can attest to particular qualities of the build, such as the build environment, build steps or builder identity.
+implementation: >
+  This probe checks how many packages published by the repository are associated with verified SLSA provenance attestations. It uses data from a ProjectPackageClient, which associates a GitHub/GitLab project with a package in a package manager. Using the data from the package manager (whom we rely on to verify the provenance attestation), this probe returns a finding for each release. For now, only NPM is supported.
+outcome:
+  - For each release, the probe returns OutcomeTrue or OutcomeFalse, depending on if the package has a verified provenance attestation.
+  - If we didn't find a package or didn't find releases, return OutcomeNotAvailable.
+remediation:
+  onOutcome: False
+  effort: Low
+  text:
+  - For NPM, publish provenance alongside your package using the `--provenance` flag (See (Introducing npm package provenance)[https://github.blog/2023-04-19-introducing-npm-package-provenance/])
+ecosystem:
+  languages:
+    - javascript
+  clients:
+    - github
+    - gitlab

probes/releasesHaveVerifiedProvenance/impl.go

@@ -0,0 +1,70 @@
+// Copyright 2024 OpenSSF Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//nolint:stylecheck
+package releasesHaveVerifiedProvenance
+
+import (
+	"embed"
+	"fmt"
+
+	"github.com/ossf/scorecard/v5/checker"
+	"github.com/ossf/scorecard/v5/finding"
+	"github.com/ossf/scorecard/v5/internal/probes"
+)
+
+func init() {
+	probes.MustRegister(Probe, Run, []probes.CheckName{probes.SignedReleases})
+}
+
+//go:embed *.yml
+var fs embed.FS
+
+const (
+	Probe = "releasesHaveVerifiedProvenance"
+)
+
+func Run(raw *checker.RawResults) ([]finding.Finding, string, error) {
+	var findings []finding.Finding
+
+	if len(raw.SignedReleasesResults.Packages) == 0 {
+		f, err := finding.NewNotApplicable(fs, Probe, "no package manager releases found", nil)
+		if err != nil {
+			return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err)
+		}
+		findings = append(findings, *f)
+		return findings, Probe, nil
+	}
+
+	for i := range raw.SignedReleasesResults.Packages {
+		p := raw.SignedReleasesResults.Packages[i]
+
+		if !p.Provenance.IsVerified {
+			f, err := finding.NewFalse(fs, Probe, "release without verified provenance", nil)
+			if err != nil {
+				return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err)
+			}
+			findings = append(findings, *f)
+			continue
+		}
+
+		f, err := finding.NewTrue(fs, Probe, "release with verified provenance", nil)
+		if err != nil {
+			return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err)
+		}
+		findings = append(findings, *f)
+	}
+
+	return findings, Probe, nil
+}

probes/releasesHaveVerifiedProvenance/impl_test.go

@@ -0,0 +1,121 @@
+// Copyright 2024 OpenSSF Scorecard Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//nolint:stylecheck
+package releasesHaveVerifiedProvenance
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/ossf/scorecard/v5/checker"
+	"github.com/ossf/scorecard/v5/finding"
+)
+
+func Test_Run(t *testing.T) {
+	t.Parallel()
+	//nolint:govet
+	tests := []struct {
+		desc     string
+		pkgs     []checker.ProjectPackage
+		outcomes []finding.Outcome
+		err      error
+	}{
+		{
+			desc:     "no packages found",
+			outcomes: []finding.Outcome{finding.OutcomeNotApplicable},
+		},
+		{
+			desc: "some releases with verified provenance",
+			pkgs: []checker.ProjectPackage{
+				{
+					Name:       "a",
+					Version:    "1.0.0",
+					Provenance: checker.PackageProvenance{IsVerified: true},
+				},
+				{
+					Name:    "a",
+					Version: "1.0.1",
+				},
+			},
+			outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeFalse},
+		},
+		{
+			desc: "all releases with verified provenance",
+			pkgs: []checker.ProjectPackage{
+				{
+					Name:       "a",
+					Version:    "1.0.0",
+					Provenance: checker.PackageProvenance{IsVerified: true},
+				},
+				{
+					Name:       "a",
+					Version:    "1.0.1",
+					Provenance: checker.PackageProvenance{IsVerified: true},
+				},
+			},
+			outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeTrue},
+		},
+		{
+			desc: "no verified provenance",
+			pkgs: []checker.ProjectPackage{
+				{
+					Name:    "a",
+					Version: "1.0.0",
+				},
+				{
+					Name:    "a",
+					Version: "1.0.1",
+				},
+			},
+			outcomes: []finding.Outcome{finding.OutcomeFalse, finding.OutcomeFalse},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt // Re-initializing variable so it is not changed while executing the closure below
+		t.Run(tt.desc, func(t *testing.T) {
+			t.Parallel()
+			raw := checker.RawResults{
+				SignedReleasesResults: checker.SignedReleasesData{
+					Packages: tt.pkgs,
+				},
+			}
+
+			outcomes, _, err := Run(&raw)
+
+			if !errors.Is(tt.err, err) {
+				t.Errorf("expected %+v got %+v", tt.err, err)
+			}
+
+			if !cmpOutcomes(tt.outcomes, outcomes) {
+				t.Errorf("expected %+v got %+v", tt.outcomes, outcomes)
+			}
+		})
+	}
+}
+
+func cmpOutcomes(ex []finding.Outcome, act []finding.Finding) bool {
+	if len(ex) != len(act) {
+		return false
+	}
+
+	for i := range ex {
+		if act[i].Outcome != ex[i] {
+			return false
+		}
+	}
+
+	return true
+}