From 4302af693e31e8dd109d34cea0a083bfaa484ca0 Mon Sep 17 00:00:00 2001 From: Avishay Balter Date: Fri, 7 Jun 2024 07:58:03 +0000 Subject: [PATCH 01/15] Nuget lock file support Signed-off-by: balteraivshay --- checks/raw/pinned_dependencies_test.go | 4 +- checks/raw/shell_download_validate.go | 113 +++++++++++-- checks/raw/shell_download_validate_test.go | 149 +++++++++++++++++- .../github-workflow-pkg-managers.yaml | 10 ++ checks/raw/testdata/Dockerfile-pkg-managers | 6 + checks/raw/testdata/script-pkg-managers | 4 +- 6 files changed, 272 insertions(+), 14 deletions(-) diff --git a/checks/raw/pinned_dependencies_test.go b/checks/raw/pinned_dependencies_test.go index c86ed60a6db..ec95026546b 100644 --- a/checks/raw/pinned_dependencies_test.go +++ b/checks/raw/pinned_dependencies_test.go @@ -281,7 +281,7 @@ func TestGithubWorkflowPkgManagerPinning(t *testing.T) { { name: "npm packages without verification", filename: "./testdata/.github/workflows/github-workflow-pkg-managers.yaml", - unpinned: 49, + unpinned: 52, }, { name: "Can't identify OS but doesn't crash", @@ -1409,7 +1409,7 @@ func TestDockerfileScriptDownload(t *testing.T) { { name: "pkg managers", filename: "./testdata/Dockerfile-pkg-managers", - unpinned: 60, + unpinned: 63, }, { name: "download with some python", diff --git a/checks/raw/shell_download_validate.go b/checks/raw/shell_download_validate.go index ee962f3ea4f..8f59aba04e5 100644 --- a/checks/raw/shell_download_validate.go +++ b/checks/raw/shell_download_validate.go @@ -764,19 +764,19 @@ func isUnpinnedNugetCliInstall(cmd []string) bool { return unpinnedDependency } -func isDotNetCliInstall(cmd []string) bool { +func isDotNetCliAdd(cmd []string) bool { // Search for command of type dotnet add package if len(cmd) < 4 { return false } - // Search for dotnet add package + // Search for dotnet add [PROJECT] package // where package command can be either the second or the third word return (isBinaryName("dotnet", cmd[0]) || isBinaryName("dotnet.exe", cmd[0])) && strings.EqualFold(cmd[1], "add") && (strings.EqualFold(cmd[2], "package") || strings.EqualFold(cmd[3], "package")) } -func isUnpinnedDotNetCliInstall(cmd []string) bool { +func isUnpinnedDotNetCliAdd(cmd []string) bool { unpinnedDependency := true for i := 3; i < len(cmd); i++ { // look for version flag @@ -789,12 +789,12 @@ func isUnpinnedDotNetCliInstall(cmd []string) bool { return unpinnedDependency } -func isNugetDownload(cmd []string) bool { - return isDotNetCliInstall(cmd) || isNugetCliInstall(cmd) +func isNuget(cmd []string) bool { + return isDotNetCliAdd(cmd) || isNugetCliInstall(cmd) || isDotNetCliRestore(cmd) || isNugetCliRestore(cmd) || isMsBuildRestore(cmd) } -func isNugetUnpinnedDownload(cmd []string) bool { - if isDotNetCliInstall(cmd) && isUnpinnedDotNetCliInstall(cmd) { +func isNugetUnpinned(cmd []string) bool { + if isDotNetCliAdd(cmd) && isUnpinnedDotNetCliAdd(cmd) { return true } @@ -802,9 +802,101 @@ func isNugetUnpinnedDownload(cmd []string) bool { return true } + if isDotNetCliRestore(cmd) && isUnpinnedDotNetCliRestore(cmd) { + return true + } + + if isNugetCliRestore(cmd) && isUnpinnedNugetCliRestore(cmd) { + return true + } + + if isMsBuildRestore(cmd) && isUnpinnedMsBuildCliRestore(cmd) { + return true + } + + return false +} + +func isNugetCliRestore(cmd []string) bool { + // Search for command of type nuget restore + if len(cmd) < 2 { + return false + } + // Search for nuget restore + return (isBinaryName("nuget", cmd[0]) || isBinaryName("nuget.exe", cmd[0])) && + strings.EqualFold(cmd[1], "restore") +} + +func isDotNetCliRestore(cmd []string) bool { + // Search for command of type dotnet restore + if len(cmd) < 2 { + return false + } + // Search for dotnet restore + return (isBinaryName("dotnet", cmd[0]) || isBinaryName("dotnet.exe", cmd[0])) && + strings.EqualFold(cmd[1], "restore") +} + +func isMsBuildRestore(cmd []string) bool { + // Search for command of type msbuild /t:restore + if len(cmd) < 2 { + return false + } + // Search for msbuild /t:restore + if isBinaryName("msbuild", cmd[0]) || isBinaryName("msbuild.exe", cmd[0]) { + for i := 1; i < len(cmd); i++ { + // look for /t:restore flag + if strings.EqualFold(cmd[i], "/t:restore") { + return true + } + } + } return false } +func isNugetRestore(cmd []string) bool { + return isDotNetCliRestore(cmd) || isNugetCliRestore(cmd) || isMsBuildRestore(cmd) +} + +func isUnpinnedNugetCliRestore(cmd []string) bool { + unpinnedDependency := true + for i := 2; i < len(cmd); i++ { + // look for LockedMode flag + // https://learn.microsoft.com/en-us/nuget/reference/cli-reference/cli-ref-restore + if strings.EqualFold(cmd[i], "-LockedMode") { + unpinnedDependency = false + break + } + } + return unpinnedDependency +} + +func isUnpinnedDotNetCliRestore(cmd []string) bool { + unpinnedDependency := true + for i := 2; i < len(cmd); i++ { + // look for locked-mode flag + // https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-restore + if strings.EqualFold(cmd[i], "--locked-mode") { + unpinnedDependency = false + break + } + } + return unpinnedDependency +} + +func isUnpinnedMsBuildCliRestore(cmd []string) bool { + unpinnedDependency := true + for i := 2; i < len(cmd); i++ { + // look for /p:RestoreLockedMode=true + // https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-restore + if strings.EqualFold(cmd[i], "/p:RestoreLockedMode=true") { + unpinnedDependency = false + break + } + } + return unpinnedDependency +} + func collectUnpinnedPackageManagerDownload(startLine, endLine uint, node syntax.Node, cmd, pathfn string, r *checker.PinningDependenciesData, ) { @@ -900,8 +992,8 @@ func collectUnpinnedPackageManagerDownload(startLine, endLine uint, node syntax. return } - // Nuget install. - if isNugetDownload(c) { + // Nuget install and restore + if isNuget(c) { r.Dependencies = append(r.Dependencies, checker.Dependency{ Location: &checker.File{ @@ -911,13 +1003,14 @@ func collectUnpinnedPackageManagerDownload(startLine, endLine uint, node syntax. EndOffset: endLine, Snippet: cmd, }, - Pinned: asBoolPointer(!isNugetUnpinnedDownload(c)), + Pinned: asBoolPointer(!isNugetUnpinned(c)), Type: checker.DependencyUseTypeNugetCommand, }, ) return } + // TODO(laurent): add other package managers. } diff --git a/checks/raw/shell_download_validate_test.go b/checks/raw/shell_download_validate_test.go index 8b644c40914..7a83a5ee327 100644 --- a/checks/raw/shell_download_validate_test.go +++ b/checks/raw/shell_download_validate_test.go @@ -128,11 +128,95 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: true, }, + { + name: "nuget.exe install", + args: args{ + cmd: []string{"nuget.exe", "install", "Newtonsoft.Json"}, + }, + want: true, + }, { name: "nuget restore", args: args{ cmd: []string{"nuget", "restore"}, }, + want: true, + }, + { + name: "nuget.exe restore", + args: args{ + cmd: []string{"nuget.exe", "restore"}, + }, + want: true, + }, + { + name: "msbuild restore", + args: args{ + cmd: []string{"msbuild", "/t:restore"}, + }, + want: true, + }, + { + name: "msbuild.exe restore", + args: args{ + cmd: []string{"msbuild.exe", "/t:restore"}, + }, + want: true, + }, + { + name: "nuget restore locked", + args: args{ + cmd: []string{"nuget", "restore", "-LockedMode"}, + }, + want: false, + }, + { + name: "nuget.exe restore locked", + args: args{ + cmd: []string{"nuget.exe", "restore", "-LockedMode"}, + }, + want: false, + }, + { + name: "msbuild restore locked", + args: args{ + cmd: []string{"msbuild", "/t:restore", "/p:RestoreLockedMode=true"}, + }, + want: false, + }, + { + name: "msbuild.exe restore locked", + args: args{ + cmd: []string{"msbuild.exe", "/t:restore", "/p:RestoreLockedMode=true"}, + }, + want: false, + }, + { + name: "dotnet restore", + args: args{ + cmd: []string{"dotnet", "restore"}, + }, + want: true, + }, + { + name: "dotnet.exe restore", + args: args{ + cmd: []string{"dotnet.exe", "restore"}, + }, + want: true, + }, + { + name: "dotnet restore locked", + args: args{ + cmd: []string{"dotnet", "restore", "--locked-mode"}, + }, + want: false, + }, + { + name: "dotnet.exe restore locked", + args: args{ + cmd: []string{"dotnet.exe", "restore", "--locked-mode"}, + }, want: false, }, { @@ -142,6 +226,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: false, }, + { + name: "nuget.exe install with -Version", + args: args{ + cmd: []string{"nuget.exe", "install", "Newtonsoft.Json", "-Version", "2"}, + }, + want: false, + }, { name: "nuget install with packages.config", args: args{ @@ -149,6 +240,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: false, }, + { + name: "nuget.exe install with packages.config", + args: args{ + cmd: []string{"nuget.exe", "install", "config\\packages.config"}, + }, + want: false, + }, { name: "dotnet add", args: args{ @@ -156,6 +254,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: true, }, + { + name: "dotnet.exe add", + args: args{ + cmd: []string{"dotnet.exe", "add", "package", "Newtonsoft.Json"}, + }, + want: true, + }, { name: "dotnet add to project", args: args{ @@ -163,6 +268,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: true, }, + { + name: "dotnet.exe add to project", + args: args{ + cmd: []string{"dotnet.exe", "add", "project1", "package", "Newtonsoft.Json"}, + }, + want: true, + }, { name: "dotnet add reference to project", args: args{ @@ -170,6 +282,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: false, }, + { + name: "dotnet.exe add reference to project", + args: args{ + cmd: []string{"dotnet.exe", "add", "project1", "reference", "OtherProject"}, + }, + want: false, + }, { name: "dotnet build", args: args{ @@ -184,6 +303,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: false, }, + { + name: "dotnet.exe add with -v", + args: args{ + cmd: []string{"dotnet.exe", "add", "package", "Newtonsoft.Json", "-v", "2.0"}, + }, + want: false, + }, { name: "dotnet add to project with -v", args: args{ @@ -191,6 +317,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: false, }, + { + name: "dotnet.exe add to project with -v", + args: args{ + cmd: []string{"dotnet.exe", "add", "project1", "package", "Newtonsoft.Json", "-v", "2.0"}, + }, + want: false, + }, { name: "dotnet add reference to project with -v", args: args{ @@ -198,6 +331,13 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: false, }, + { + name: "dotnet.exe add reference to project with -v", + args: args{ + cmd: []string{"dotnet.exe", "add", "project1", "reference", "Newtonsoft.Json", "-v", "2.0"}, + }, + want: false, + }, { name: "dotnet add with --version", args: args{ @@ -205,12 +345,19 @@ func Test_isDotNetUnpinnedDownload(t *testing.T) { }, want: false, }, + { + name: "dotnet.exe add with --version", + args: args{ + cmd: []string{"dotnet.exe", "add", "package", "Newtonsoft.Json", "--version", "2.0"}, + }, + want: false, + }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - if got := isNugetUnpinnedDownload(tt.args.cmd); got != tt.want { + if got := isNugetUnpinned(tt.args.cmd); got != tt.want { t.Errorf("isNugetUnpinnedDownload() = %v, want %v", got, tt.want) } }) diff --git a/checks/raw/testdata/.github/workflows/github-workflow-pkg-managers.yaml b/checks/raw/testdata/.github/workflows/github-workflow-pkg-managers.yaml index 7b28e3ec97c..f4f300ff9e9 100644 --- a/checks/raw/testdata/.github/workflows/github-workflow-pkg-managers.yaml +++ b/checks/raw/testdata/.github/workflows/github-workflow-pkg-managers.yaml @@ -166,6 +166,16 @@ jobs: run: nuget install 'some-package' - name: run: nuget restore + - name: + run: nuget restore -LockedMode + - name: + run: dotnet restore + - name: + run: dotnet restore --locked-mode + - name: + run: msbuild /t:restore /p:RestoreLockedMode=true + - name: + run: msbuild /t:restore - name: run: dotnet add package 'some-package' - name: diff --git a/checks/raw/testdata/Dockerfile-pkg-managers b/checks/raw/testdata/Dockerfile-pkg-managers index 0208433c8d5..72b031027bb 100644 --- a/checks/raw/testdata/Dockerfile-pkg-managers +++ b/checks/raw/testdata/Dockerfile-pkg-managers @@ -127,6 +127,12 @@ RUN choco install --require-checksums 'some-package' RUN nuget install some-package RUN nuget restore +RUN nuget restore -LockedMode +RUN dotnet restore +RUN dotnet restore --locked-mode +RUN msbuild.exe /t:restore /p:RestoreLockedMode=true +RUN msbuild.exe /t:restore +RUN nuget restore -LockedMode RUN nuget install some-package -Version 1.2.3 RUN nuget install packages.config RUN dotnet add package some-package diff --git a/checks/raw/testdata/script-pkg-managers b/checks/raw/testdata/script-pkg-managers index e13e6f2b646..4041020c3ac 100644 --- a/checks/raw/testdata/script-pkg-managers +++ b/checks/raw/testdata/script-pkg-managers @@ -125,7 +125,9 @@ choco install --requirechecksums 'some-package' choco install --require-checksums 'some-package' nuget install some-package -nuget restore +nuget restore -LockedMode +dotnet restore --locked-mode +msbuild.exe /t:restore /p:RestoreLockedMode=true nuget install some-package -Version 1.2.3 nuget install packages.config dotnet add package some-package From 4c7e4f418819e8889a59052a50f0ca95772d4a7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:25:36 -0700 Subject: [PATCH 02/15] :seedling: Bump github.com/google/osv-scanner from 1.7.3 to 1.7.4 (#4139) Bumps [github.com/google/osv-scanner](https://github.com/google/osv-scanner) from 1.7.3 to 1.7.4. - [Release notes](https://github.com/google/osv-scanner/releases) - [Changelog](https://github.com/google/osv-scanner/blob/main/CHANGELOG.md) - [Commits](https://github.com/google/osv-scanner/compare/v1.7.3...v1.7.4) --- updated-dependencies: - dependency-name: github.com/google/osv-scanner dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: balteraivshay --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 312d7f4289c..5044040e314 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/caarlos0/env/v6 v6.10.0 github.com/gobwas/glob v0.2.3 github.com/google/go-github/v53 v53.2.0 - github.com/google/osv-scanner v1.7.3 + github.com/google/osv-scanner v1.7.4 github.com/mcuadros/go-jsonschema-generator v0.0.0-20200330054847-ba7a369d4303 github.com/onsi/ginkgo/v2 v2.19.0 github.com/otiai10/copy v1.14.0 @@ -56,8 +56,8 @@ require ( cloud.google.com/go/containeranalysis v0.11.5 // indirect cloud.google.com/go/kms v1.15.8 // indirect dario.cat/mergo v1.0.0 // indirect - deps.dev/api/v3 v3.0.0-20240503042720-6166138ce783 // indirect - github.com/BurntSushi/toml v1.3.2 // indirect + deps.dev/api/v3 v3.0.0-20240516073147-b352d7eeeae6 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/CycloneDX/cyclonedx-go v0.8.0 // indirect github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect @@ -93,7 +93,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/owenrumney/go-sarif/v2 v2.3.1 // indirect - github.com/package-url/packageurl-go v0.1.2 // indirect + github.com/package-url/packageurl-go v0.1.3 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -175,13 +175,13 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect + golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.20.0 golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.177.0 // indirect - google.golang.org/grpc v1.63.2 // indirect + google.golang.org/grpc v1.64.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index b1d0452e0f7..a04c8183166 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ contrib.go.opencensus.io/exporter/stackdriver v0.13.14 h1:zBakwHardp9Jcb8sQHcHpX contrib.go.opencensus.io/exporter/stackdriver v0.13.14/go.mod h1:5pSSGY0Bhuk7waTHuDf4aQ8D2DrhgETRo9fy6k3Xlzc= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -deps.dev/api/v3 v3.0.0-20240503042720-6166138ce783 h1:ls9MrAHgWyVrhaRw/xGuFc6/oNUR4VYKGWf2ZCAVsjo= -deps.dev/api/v3 v3.0.0-20240503042720-6166138ce783/go.mod h1:DyBY3wNVqRCwvb4tLvz6LL/FupH3FMflEROyQAv2Vi0= +deps.dev/api/v3 v3.0.0-20240516073147-b352d7eeeae6 h1:pxD1B50Np8PJRcI3KkYHWCfbhOSuDKz1tzXoazCO8eg= +deps.dev/api/v3 v3.0.0-20240516073147-b352d7eeeae6/go.mod h1:DyBY3wNVqRCwvb4tLvz6LL/FupH3FMflEROyQAv2Vi0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -83,8 +83,8 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CycloneDX/cyclonedx-go v0.8.0 h1:FyWVj6x6hoJrui5uRQdYZcSievw3Z32Z88uYzG/0D6M= github.com/CycloneDX/cyclonedx-go v0.8.0/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= @@ -406,8 +406,8 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/osv-scanner v1.7.3 h1:zT7g8QkYag0c5a/OmPXmiHuCldj9gWcF4OmcPFFp7oY= -github.com/google/osv-scanner v1.7.3/go.mod h1:uRrd+koUIBfw1fLAG2Zfyql0cz5HGfKB1e2RdGz/fPs= +github.com/google/osv-scanner v1.7.4 h1:u021ANn9P/xZUPpYXVv7O1n9FYhF+YmYEqt/05BLpPA= +github.com/google/osv-scanner v1.7.4/go.mod h1:GWXnZYfJEeICRUFJIbW/BRHzsDHVkqPOvKo5uBi5+jU= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -628,8 +628,8 @@ github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAq github.com/owenrumney/go-sarif v1.1.1/go.mod h1:dNDiPlF04ESR/6fHlPyq7gHKmrM0sHUvAGjsoh8ZH0U= github.com/owenrumney/go-sarif/v2 v2.3.1 h1:77opmuqxQZE1UF6TylFz5XllVEI72WijgwpwNw4JTmY= github.com/owenrumney/go-sarif/v2 v2.3.1/go.mod h1:MSqMMx9WqlBSY7pXoOZWgEsVB4FDNfhcaXDA1j6Sr+w= -github.com/package-url/packageurl-go v0.1.2 h1:0H2DQt6DHd/NeRlVwW4EZ4oEI6Bn40XlNPRqegcxuo4= -github.com/package-url/packageurl-go v0.1.2/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= +github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs= +github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= github.com/pandatix/go-cvss v0.6.2 h1:TFiHlzUkT67s6UkelHmK6s1INKVUG7nlKYiWWDTITGI= github.com/pandatix/go-cvss v0.6.2/go.mod h1:jDXYlQBZrc8nvrMUVVvTG8PhmuShOnKrxP53nOFkt8Q= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -848,8 +848,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d h1:N0hmiNbwsSNwHBAvR3QB5w25pUwH4tK0Y/RltD1j1h4= +golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1164,8 +1164,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 4af64633ff7f931a340008338b51a5cba12fefcb Mon Sep 17 00:00:00 2001 From: aklevans <105876795+aklevans@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:15:29 -0400 Subject: [PATCH 03/15] =?UTF-8?q?=F0=9F=90=9B=20Use=20direct=20endpoint=20?= =?UTF-8?q?instead=20of=20search=20to=20find=20repository=20URL=20from=20n?= =?UTF-8?q?pm=20database=20(#4118)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update endpoint used when getting repo from npm to solve #3166 Signed-off-by: aklevans * Update test files to account for endpoint change when getting repo from npm Signed-off-by: aklevans * Fix linter issues Signed-off-by: aklevans * Added unit tests for #3166 and #2441 Signed-off-by: aklevans * fix linter issues and reduce mock json output in package_manager_test to only include necessary data Signed-off-by: aklevans * fix linter issues in package_managers.go Signed-off-by: aklevans * convert windows line breaks to linux Signed-off-by: aklevans * reduce test case size, still has windows line breaks Signed-off-by: aklevans * Fix unit tests Signed-off-by: aklevans * attempt linter fix Signed-off-by: aklevans * Fix linter issues stemming from windows line breaks Signed-off-by: aklevans * Remove magic number and rename variable to be more accurate Signed-off-by: aklevans --------- Signed-off-by: aklevans Signed-off-by: aklevans <105876795+aklevans@users.noreply.github.com> Signed-off-by: balteraivshay --- cmd/package_managers.go | 24 +++++------ cmd/package_managers_test.go | 83 +++++++++++++++++------------------- 2 files changed, 51 insertions(+), 56 deletions(-) diff --git a/cmd/package_managers.go b/cmd/package_managers.go index 302a44dc108..889fabd9628 100644 --- a/cmd/package_managers.go +++ b/cmd/package_managers.go @@ -19,6 +19,7 @@ import ( "encoding/json" "fmt" "io" + "net/http" "regexp" "strings" @@ -106,14 +107,10 @@ func fetchGitRepositoryFromPackageManagers(npm, pypi, rubygems, nuget string, return packageMangerResponse{}, nil } -type npmSearchResults struct { - Objects []struct { - Package struct { - Links struct { - Repository string `json:"repository"` - } `json:"links"` - } `json:"package"` - } `json:"objects"` +type npmResult struct { + Repository struct { + URL string `json:"url"` + } `json:"repository"` } type pypiSearchResults struct { @@ -129,23 +126,24 @@ type rubyGemsSearchResults struct { // Gets the GitHub repository URL for the npm package. func fetchGitRepositoryFromNPM(packageName string, packageManager pmc.Client) (string, error) { - npmSearchURL := "https://registry.npmjs.org/-/v1/search?text=%s&size=1" - resp, err := packageManager.Get(npmSearchURL, packageName) + npmGetURL := "https://registry.npmjs.org/%s/latest" + + resp, err := packageManager.Get(npmGetURL, packageName) if err != nil { return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("failed to get npm package json: %v", err)) } defer resp.Body.Close() - v := &npmSearchResults{} + v := &npmResult{} err = json.NewDecoder(resp.Body).Decode(v) if err != nil { return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("failed to parse npm package json: %v", err)) } - if len(v.Objects) == 0 { + if resp.StatusCode == http.StatusNotFound || v.Repository.URL == "" { return "", sce.WithMessage(sce.ErrScorecardInternal, fmt.Sprintf("could not find source repo for npm package: %s", packageName)) } - return v.Objects[0].Package.Links.Repository, nil + return strings.TrimPrefix(strings.TrimSuffix(v.Repository.URL, ".git"), "git+"), nil } func findGitRepositoryInPYPIResponse(packageName string, response io.Reader) (string, error) { diff --git a/cmd/package_managers_test.go b/cmd/package_managers_test.go index f41cf3381cc..5ee5d4681bb 100644 --- a/cmd/package_managers_test.go +++ b/cmd/package_managers_test.go @@ -48,53 +48,48 @@ func Test_fetchGitRepositoryFromNPM(t *testing.T) { packageName: "npm-package", result: ` { - "objects": [ - { - "package": { - "name": "@pulumi/pulumi", - "scope": "pulumi", - "version": "3.26.0", - "description": "Pulumi's Node.js SDK", - "date": "2022-03-09T14:05:40.682Z", - "links": { - "homepage": "https://github.com/pulumi/pulumi#readme", - "repository": "https://github.com/pulumi/pulumi", - "bugs": "https://github.com/pulumi/pulumi/issues" - }, - "publisher": { - "username": "pulumi-bot", - "email": "bot@pulumi.com" - }, - "maintainers": [ - { - "username": "joeduffy", - "email": "joe@pulumi.com" - }, - { - "username": "pulumi-bot", - "email": "bot@pulumi.com" - } - ] - }, - "score": { - "final": 0.4056031974977145, - "detail": { - "quality": 0.7308571951451065, - "popularity": 0.19908392082147397, - "maintenance": 0.3333333333333333 - } - }, - "searchScore": 0.00090895034 - } - ], - "total": 380, - "time": "Wed Mar 09 2022 18:11:10 GMT+0000 (Coordinated Universal Time)" + "name": "@pulumi/pulumi", + "version": "3.116.1", + "description": "Pulumi's Node.js SDK", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/pulumi/pulumi.git", + "directory": "sdk/nodejs" + } + } `, }, want: "https://github.com/pulumi/pulumi", wantErr: false, }, + { + name: "fetchGitRepositoryFromNPM", + + args: args{ + packageName: "left-pad", + result: ` +{ + "name": "left-pad", + "version": "1.3.0", + "description": "String left pad", + "main": "index.js", + "types": "index.d.ts", + "scripts": { + "test": "node test", + "bench": "node perf/perf.js" + }, + "repository": { + "url": "git+ssh://git@github.com/stevemao/left-pad.git", + "type": "git" + } +} + `, + }, + want: "ssh://git@github.com/stevemao/left-pad", + wantErr: false, + }, { name: "fetchGitRepositoryFromNPM_error", @@ -109,8 +104,10 @@ func Test_fetchGitRepositoryFromNPM(t *testing.T) { name: "fetchGitRepositoryFromNPM_error", args: args{ - packageName: "npm-package", - result: "foo", + packageName: "https://github.com/airbnb/lottie-web", + result: ` + {"code":"ResourceNotFound","message":"/https:/github.com/airbnb/lottie-web does not exist"} + `, }, want: "", wantErr: true, From a8cb31d4e7c9a0f6968e12c99bc4ca37b6b15e41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:44:34 +0000 Subject: [PATCH 04/15] :seedling: Bump golang.org/x/text from 0.15.0 to 0.16.0 (#4142) Signed-off-by: balteraivshay --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5044040e314..464ceced5d0 100644 --- a/go.mod +++ b/go.mod @@ -29,8 +29,8 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 go.opencensus.io v0.24.0 gocloud.dev v0.37.0 - golang.org/x/text v0.15.0 - golang.org/x/tools v0.21.0 // indirect + golang.org/x/text v0.16.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/protobuf v1.34.1 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index a04c8183166..c21ebb9380d 100644 --- a/go.sum +++ b/go.sum @@ -1035,8 +1035,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1091,8 +1091,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/vuln v1.0.4 h1:SP0mPeg2PmGCu03V+61EcQiOjmpri2XijexKdzv8Z1I= golang.org/x/vuln v1.0.4/go.mod h1:NbJdUQhX8jY++FtuhrXs2Eyx0yePo9pF7nPlIjo9aaQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From ead508dc0f53f0c459d898bcaed719e541a27c0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 18:00:32 +0000 Subject: [PATCH 05/15] :seedling: Bump github.com/rhysd/actionlint from 1.7.0 to 1.7.1 (#4138) Signed-off-by: balteraivshay --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 464ceced5d0..4ac23d2b70b 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/moby/buildkit v0.13.2 github.com/olekukonko/tablewriter v0.0.5 github.com/onsi/gomega v1.33.1 - github.com/rhysd/actionlint v1.7.0 + github.com/rhysd/actionlint v1.7.1 github.com/shurcooL/githubv4 v0.0.0-20201206200315-234843c633fa github.com/shurcooL/graphql v0.0.0-20200928012149-18c5c3165e3a github.com/sirupsen/logrus v1.9.3 @@ -141,7 +141,7 @@ require ( github.com/docker/docker v25.0.5+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect - github.com/fatih/color v1.16.0 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index c21ebb9380d..e65f7fb757e 100644 --- a/go.sum +++ b/go.sum @@ -257,8 +257,8 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -666,8 +666,8 @@ github.com/prometheus/prometheus v0.50.1 h1:N2L+DYrxqPh4WZStU+o1p/gQlBaqFbcLBTjl github.com/prometheus/prometheus v0.50.1/go.mod h1:FvE8dtQ1Ww63IlyKBn1V4s+zMwF9kHkVNkQBR1pM4CU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/rhysd/actionlint v1.7.0 h1:xRSXsT5GGwWTpabqtksigZf27G06VOf7kyEGezYum1w= -github.com/rhysd/actionlint v1.7.0/go.mod h1:Pkvt8R+wpr8LQAAEE2xv9hhdPufPkkAwnh+QFydomqg= +github.com/rhysd/actionlint v1.7.1 h1:WJaDzyT1StBWVKGSsZPYnbV0HF9Y9/vD6KFdZQL42qE= +github.com/rhysd/actionlint v1.7.1/go.mod h1:lNjNNlZY0BdBl8l837Z9ZiBpu8v+5lzfoJQFdSk4xss= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= From 12f1ca5cd8383dbc75fe6f6b76077c76085f7c71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 18:13:00 +0000 Subject: [PATCH 06/15] :seedling: Bump github.com/bradleyfalzon/ghinstallation/v2 (#4137) Signed-off-by: balteraivshay --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 4ac23d2b70b..e330a893880 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( cloud.google.com/go/trace v1.10.6 // indirect contrib.go.opencensus.io/exporter/stackdriver v0.13.14 github.com/bombsimon/logrusr/v2 v2.0.1 - github.com/bradleyfalzon/ghinstallation/v2 v2.10.0 + github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 github.com/go-git/go-git/v5 v5.12.0 github.com/go-logr/logr v1.4.2 github.com/golang/mock v1.6.0 @@ -77,7 +77,7 @@ require ( github.com/golang/glog v1.2.0 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-github/v60 v60.0.0 // indirect + github.com/google/go-github/v62 v62.0.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/google/s2a-go v0.1.7 // indirect diff --git a/go.sum b/go.sum index e65f7fb757e..3bb23e47a80 100644 --- a/go.sum +++ b/go.sum @@ -166,8 +166,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM= github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio= -github.com/bradleyfalzon/ghinstallation/v2 v2.10.0 h1:XWuWBRFEpqVrHepQob9yPS3Xg4K3Wr9QCx4fu8HbUNg= -github.com/bradleyfalzon/ghinstallation/v2 v2.10.0/go.mod h1:qoGA4DxWPaYTgVCrmEspVSjlTu4WYAiSxMIhorMRXXc= +github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53AL4ED5MzgEMwGTZag= +github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -389,8 +389,8 @@ github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= -github.com/google/go-github/v60 v60.0.0 h1:oLG98PsLauFvvu4D/YPxq374jhSxFYdzQGNCyONLfn8= -github.com/google/go-github/v60 v60.0.0/go.mod h1:ByhX2dP9XT9o/ll2yXAu2VD8l5eNVg8hD4Cr0S/LmQk= +github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= +github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-replayers/grpcreplay v1.1.0 h1:S5+I3zYyZ+GQz68OfbURDdt/+cSMqCK1wrvNx7WBzTE= From 8de4b870781f3fd2e94c96c85259f97269ad6517 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Thu, 6 Jun 2024 09:47:06 -0700 Subject: [PATCH 07/15] :warning: remove dependencydiff functionality (#4146) Signed-off-by: Spencer Schrock Signed-off-by: balteraivshay --- .codecov.yml | 2 - dependencydiff/dependencydiff.go | 206 ----------------------- dependencydiff/dependencydiff_test.go | 232 -------------------------- dependencydiff/errors.go | 23 --- dependencydiff/mapping.go | 120 ------------- dependencydiff/raw_dependencies.go | 77 --------- e2e/dependencydiff_test.go | 93 ----------- 7 files changed, 753 deletions(-) delete mode 100644 dependencydiff/dependencydiff.go delete mode 100644 dependencydiff/dependencydiff_test.go delete mode 100644 dependencydiff/errors.go delete mode 100644 dependencydiff/mapping.go delete mode 100644 dependencydiff/raw_dependencies.go delete mode 100644 e2e/dependencydiff_test.go diff --git a/.codecov.yml b/.codecov.yml index 97cae9da52b..01a294fe37a 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -12,8 +12,6 @@ ignore: - "docs/**/*" # this is the runner - "main.go" - # this package is deprecated and going to be removed. - - "dependencydiff/**/*" coverage: precision: 2 diff --git a/dependencydiff/dependencydiff.go b/dependencydiff/dependencydiff.go deleted file mode 100644 index 3eb28f837d9..00000000000 --- a/dependencydiff/dependencydiff.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2022 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Deprecated: This is going to be removed in the future. -package dependencydiff - -import ( - "context" - "fmt" - "strings" - - "github.com/ossf/scorecard/v5/checker" - "github.com/ossf/scorecard/v5/checks" - "github.com/ossf/scorecard/v5/clients" - sce "github.com/ossf/scorecard/v5/errors" - "github.com/ossf/scorecard/v5/internal/packageclient" - sclog "github.com/ossf/scorecard/v5/log" - "github.com/ossf/scorecard/v5/pkg" - "github.com/ossf/scorecard/v5/policy" -) - -// Depdiff is the exported name for dependency-diff. -const ( - Depdiff = "Dependency-diff" -) - -// A private context struct used for GetDependencyCheckResults. -type dependencydiffContext struct { - logger *sclog.Logger - ownerName, repoName, base, head string - ctx context.Context - ghRepo clients.Repo - ghRepoClient clients.RepoClient - ossFuzzClient clients.RepoClient - vulnsClient clients.VulnerabilitiesClient - ciiClient clients.CIIBestPracticesClient - projectClient packageclient.ProjectPackageClient - changeTypesToCheck []string - checkNamesToRun []string - dependencydiffs []dependency - results []pkg.DependencyCheckResult -} - -// Deprecated: This is going to be removed in the future. -// GetDependencyDiffResults gets dependency changes between two given code commits BASE and HEAD -// along with the Scorecard check results of the dependencies, and returns a slice of DependencyCheckResult. -// TO use this API, an access token must be set. See https://github.com/ossf/scorecard#authentication. -func GetDependencyDiffResults( - ctx context.Context, - repoURI string, /* Use the format "ownerName/repoName" as the repo URI, such as "ossf/scorecard". */ - base, head string, /* Two code commits base and head, can use either SHAs or branch names. */ - checksToRun []string, /* A list of enabled check names to run. */ - changeTypes []string, /* A list of dependency change types for which we surface scorecard results. */ -) ([]pkg.DependencyCheckResult, error) { - logger := sclog.NewLogger(sclog.DefaultLevel) - ownerAndRepo := strings.Split(repoURI, "/") - if len(ownerAndRepo) != 2 { - return nil, fmt.Errorf("%w: repo uri input", errInvalid) - } - owner, repo := ownerAndRepo[0], ownerAndRepo[1] - dCtx := dependencydiffContext{ - logger: logger, - ownerName: owner, - repoName: repo, - base: base, - head: head, - ctx: ctx, - changeTypesToCheck: changeTypes, - checkNamesToRun: checksToRun, - } - // Fetch the raw dependency diffs. This API will also handle error cases such as invalid base or head. - err := fetchRawDependencyDiffData(&dCtx) - if err != nil { - return nil, fmt.Errorf("error in fetchRawDependencyDiffData: %w", err) - } - // Map the ecosystem naming convention from GitHub to OSV. - err = mapDependencyEcosystemNaming(dCtx.dependencydiffs) - if err != nil { - return nil, fmt.Errorf("error in mapDependencyEcosystemNaming: %w", err) - } - err = getScorecardCheckResults(&dCtx) - if err != nil { - return nil, fmt.Errorf("error getting scorecard check results: %w", err) - } - return dCtx.results, nil -} - -func initRepoAndClientByChecks(dCtx *dependencydiffContext, dSrcRepo string) error { - repo, repoClient, ossFuzzClient, ciiClient, vulnsClient, projectClient, err := checker.GetClients( - dCtx.ctx, dSrcRepo, "", dCtx.logger) - if err != nil { - return fmt.Errorf("error getting the github repo and clients: %w", err) - } - dCtx.ghRepo = repo - dCtx.ghRepoClient = repoClient - // If the caller doesn't specify the checks to run, run all the checks and return all the clients. - if dCtx.checkNamesToRun == nil || len(dCtx.checkNamesToRun) == 0 { - dCtx.ossFuzzClient, dCtx.ciiClient, dCtx.vulnsClient = ossFuzzClient, ciiClient, vulnsClient - return nil - } - for _, cn := range dCtx.checkNamesToRun { - switch cn { - case checks.CheckFuzzing: - dCtx.ossFuzzClient = ossFuzzClient - case checks.CheckCIIBestPractices: - dCtx.ciiClient = ciiClient - case checks.CheckVulnerabilities: - dCtx.vulnsClient = vulnsClient - case checks.CheckSignedReleases: - dCtx.projectClient = projectClient - } - } - return nil -} - -func getScorecardCheckResults(dCtx *dependencydiffContext) error { - // Initialize the checks to run from the caller's input. - checksToRun, err := policy.GetEnabled(nil, dCtx.checkNamesToRun, nil) - if err != nil { - return fmt.Errorf("error init scorecard checks: %w", err) - } - for _, d := range dCtx.dependencydiffs { - depCheckResult := pkg.DependencyCheckResult{ - PackageURL: d.PackageURL, - SourceRepository: d.SourceRepository, - ChangeType: d.ChangeType, - ManifestPath: d.ManifestPath, - Ecosystem: d.Ecosystem, - Version: d.Version, - Name: d.Name, - /* The scorecard check result is nil now. */ - } - if d.ChangeType == nil { - // Since we allow a dependency having a nil change type, so we also - // give such a dependency a nil scorecard result. - dCtx.results = append(dCtx.results, depCheckResult) - continue - } - // (1) If no change types are specified, run the checks on all types of dependencies. - // (2) If there are change types specified by the user, run the checks on the specified types. - noneGivenOrIsSpecified := len(dCtx.changeTypesToCheck) == 0 || /* None specified.*/ - isSpecifiedByUser(*d.ChangeType, dCtx.changeTypesToCheck) /* Specified by the user.*/ - // For now we skip those without source repo urls. - // TODO (#2063): use the BigQuery dataset to supplement null source repo URLs to fetch the Scorecard results for them. - if d.SourceRepository != nil && noneGivenOrIsSpecified { - // Initialize the repo and client(s) corresponding to the checks to run. - err = initRepoAndClientByChecks(dCtx, *d.SourceRepository) - if err != nil { - return fmt.Errorf("error init repo and clients: %w", err) - } - - // Run scorecard on those types of dependencies that the caller would like to check. - // If the input map changeTypesToCheck is empty, by default, we run the checks for all valid types. - // TODO (#2064): use the Scorecard REST API to retrieve the Scorecard result statelessly. - scorecardResult, err := pkg.RunScorecard( - dCtx.ctx, - dCtx.ghRepo, - // TODO (#2065): In future versions, ideally, this should be - // the commitSHA corresponding to d.Version instead of HEAD. - clients.HeadSHA, - 0, - checksToRun, - dCtx.ghRepoClient, - dCtx.ossFuzzClient, - dCtx.ciiClient, - dCtx.vulnsClient, - dCtx.projectClient, - ) - // If the run fails, we leave the current dependency scorecard result empty and record the error - // rather than letting the entire API return nil since we still expect results for other dependencies. - if err != nil { - wrappedErr := sce.WithMessage(sce.ErrScorecardInternal, - fmt.Sprintf("scorecard running failed for %s: %v", d.Name, err)) - dCtx.logger.Error(wrappedErr, "") - depCheckResult.ScorecardResultWithError.Error = wrappedErr - } else { // Otherwise, we record the scorecard check results for this dependency. - depCheckResult.ScorecardResultWithError.ScorecardResult = &scorecardResult - } - } - dCtx.results = append(dCtx.results, depCheckResult) - } - return nil -} - -func isSpecifiedByUser(ct pkg.ChangeType, changeTypes []string) bool { - if len(changeTypes) == 0 { - return false - } - for _, ctByUser := range changeTypes { - if string(ct) == ctByUser { - return true - } - } - return false -} diff --git a/dependencydiff/dependencydiff_test.go b/dependencydiff/dependencydiff_test.go deleted file mode 100644 index 7172804d234..00000000000 --- a/dependencydiff/dependencydiff_test.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2022 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dependencydiff - -import ( - "context" - "errors" - "path" - "testing" - - sclog "github.com/ossf/scorecard/v5/log" - "github.com/ossf/scorecard/v5/pkg" -) - -func Test_initRepoAndClientByChecks(t *testing.T) { - t.Parallel() - //nolint:govet - tests := []struct { - name string - dCtx dependencydiffContext - srcRepo string - wantRepoClient, wantFuzzClient bool - wantVulnClient, wantCIIClient bool - wantErr bool - }{ - { - name: "error creating repo", - dCtx: dependencydiffContext{ - logger: sclog.NewLogger(sclog.InfoLevel), - ctx: context.Background(), - checkNamesToRun: []string{}, - }, - srcRepo: path.Join( - "host_not_exist.com", - "owner_not_exist", - "repo_not_exist", - ), - wantRepoClient: false, - wantFuzzClient: false, - wantVulnClient: false, - wantCIIClient: false, - wantErr: true, - }, - // Same as the above, putting the normal responses to the e2e test. - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - err := initRepoAndClientByChecks(&tt.dCtx, tt.srcRepo) - if (err != nil) != tt.wantErr { - t.Errorf("initClientByChecks() error = {%v}, want error: %v", err, tt.wantErr) - return - } - if (tt.dCtx.ghRepoClient != nil) != tt.wantRepoClient { - t.Errorf("init repo error, wantRepoClient: %v, got %v", tt.wantRepoClient, tt.dCtx.ghRepoClient) - return - } - if (tt.dCtx.ossFuzzClient != nil) != tt.wantFuzzClient { - t.Errorf("init repo error, wantFuzzClient: %v, got %v", tt.wantFuzzClient, tt.dCtx.ossFuzzClient) - return - } - if (tt.dCtx.vulnsClient != nil) != tt.wantVulnClient { - t.Errorf("init repo error, wantVulnClient: %v, got %v", tt.wantVulnClient, tt.dCtx.vulnsClient) - return - } - if (tt.dCtx.ciiClient != nil) != tt.wantCIIClient { - t.Errorf("init repo error, wantCIIClient: %v, got %v", tt.wantCIIClient, tt.dCtx.ciiClient) - return - } - }) - } -} - -func Test_getScorecardCheckResults(t *testing.T) { - t.Parallel() - tests := []struct { - name string - dCtx dependencydiffContext - wantErr bool - }{ - { - name: "empty response", - dCtx: dependencydiffContext{ - ctx: context.Background(), - logger: sclog.NewLogger(sclog.InfoLevel), - ownerName: "owner_not_exist", - repoName: "repo_not_exist", - }, - wantErr: false, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - err := getScorecardCheckResults(&tt.dCtx) - if (err != nil) != tt.wantErr { - t.Errorf("getScorecardCheckResults() error = {%v}, want error: %v", err, tt.wantErr) - return - } - }) - } -} - -func Test_mapDependencyEcosystemNaming(t *testing.T) { - t.Parallel() - //nolint:govet - tests := []struct { - name string - deps []dependency - errWanted error - }{ - { - name: "error invalid github ecosystem", - deps: []dependency{ - { - Name: "dependency_1", - Ecosystem: asPointer("not_supported"), - }, - { - Name: "dependency_2", - Ecosystem: asPointer("gomod"), - }, - }, - errWanted: errInvalid, - }, - { - name: "error cannot find mapping", - deps: []dependency{ - { - Name: "dependency_3", - Ecosystem: asPointer("foobar"), - }, - }, - errWanted: errMappingNotFound, - }, - { - name: "correct mapping", - deps: []dependency{ - { - Name: "dependency_4", - Ecosystem: asPointer("gomod"), - }, - { - Name: "dependency_5", - Ecosystem: asPointer("pip"), - }, - { - Name: "dependency_6", - Ecosystem: asPointer("cargo"), - }, - { - Name: "dependency_7", - Ecosystem: asPointer("actions"), - }, - }, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - err := mapDependencyEcosystemNaming(tt.deps) - if tt.errWanted != nil && errors.Is(tt.errWanted, err) { - t.Errorf("not a wanted error, want:%v, got:%v", tt.errWanted, err) - return - } - }) - } -} - -func Test_isSpecifiedByUser(t *testing.T) { - t.Parallel() - tests := []struct { - name string - ct pkg.ChangeType - changeTypesToCheck []string - resultWanted bool - }{ - { - name: "error invalid github ecosystem", - }, - { - name: "added", - ct: pkg.ChangeType("added"), - changeTypesToCheck: nil, - resultWanted: false, - }, - { - name: "ct is added but not specified", - ct: pkg.ChangeType("added"), - changeTypesToCheck: []string{"removed"}, - resultWanted: false, - }, - { - name: "removed", - ct: pkg.ChangeType("added"), - changeTypesToCheck: []string{"added", "removed"}, - resultWanted: true, - }, - { - name: "not_supported", - ct: pkg.ChangeType("not_supported"), - changeTypesToCheck: []string{"added", "removed"}, - resultWanted: false, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - result := isSpecifiedByUser(tt.ct, tt.changeTypesToCheck) - if result != tt.resultWanted { - t.Errorf("result (%v) != result wanted (%v)", result, tt.resultWanted) - return - } - }) - } -} diff --git a/dependencydiff/errors.go b/dependencydiff/errors.go deleted file mode 100644 index 72b57f56794..00000000000 --- a/dependencydiff/errors.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dependencydiff - -import "errors" - -// static Errors for mapping. -var ( - errMappingNotFound = errors.New("ecosystem mapping not found") - errInvalid = errors.New("invalid") -) diff --git a/dependencydiff/mapping.go b/dependencydiff/mapping.go deleted file mode 100644 index ef0a102460a..00000000000 --- a/dependencydiff/mapping.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2022 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dependencydiff - -import ( - "fmt" -) - -// Ecosystem is a package ecosystem supported by OSV, GitHub, etc. -type ecosystem string - -// OSV ecosystem naming data source: https://ossf.github.io/osv-schema/#affectedpackage-field -const ( - // The Go ecosystem. - ecosystemGo ecosystem = "Go" - - // The NPM ecosystem. - ecosystemNpm ecosystem = "npm" - - // The Android ecosystem. - ecosystemAndroid ecosystem = "Android" //nolint:unused - - // The crates.io ecosystem for RUST. - ecosystemCrates ecosystem = "crates.io" - - // For reports from the OSS-Fuzz project that have no more appropriate ecosystem. - ecosystemOssFuzz ecosystem = "OSS-Fuzz" //nolint:unused - - // The Python PyPI ecosystem. PyPI is the main package source of pip. - ecosystemPyPI ecosystem = "PyPI" - - // The RubyGems ecosystem. - ecosystemRubyGems ecosystem = "RubyGems" - - // The PHP package manager ecosystem. Packagist is the main Composer repository. - ecosystemPackagist ecosystem = "Packagist" - - // The Maven Java package ecosystem. - ecosystemMaven ecosystem = "Maven" - - // The NuGet package ecosystem. - ecosystemNuGet ecosystem = "NuGet" - - // The Linux kernel. - ecosystemLinux ecosystem = "Linux" //nolint:unused - - // The Debian package ecosystem. - ecosystemDebian ecosystem = "Debian" //nolint:unused - - // Hex is the package manager of Erlang. - // TODO: GitHub doesn't support hex as the ecosystem for Erlang yet. Add this to the map in the future. - ecosystemHex ecosystem = "Hex" //nolint:unused - - // GitHub Actions is an ecosystem for the GitHub Actions. - ecosystemActions ecosystem = "GitHub Actions" - - // Pub is the official package repository for Dart and Flutter apps. - ecosystemPub ecosystem = "Pub" //nolint:unused - - // Ecosystems with a "nolint" tag suggests GitHub hasn't gotten them supported yet. - // We need to add them to the below hashmap in a timely manner once GitHub adds supports. -) - -// gitHubToOSV defines the ecosystem naming mapping relationship between GitHub and others. -var gitHubToOSV = map[string]ecosystem{ - // GitHub ecosystem naming data source: https://docs.github.com/en/code-security/supply-chain-security/ - // understanding-your-software-supply-chain/about-the-dependency-graph#supported-package-ecosystems - "gomod": ecosystemGo, /* go.mod and go.sum */ - "cargo": ecosystemCrates, - "pip": ecosystemPyPI, /* pip and poetry */ - "npm": ecosystemNpm, /* npm and yarn */ - "maven": ecosystemMaven, - "composer": ecosystemPackagist, - "rubygems": ecosystemRubyGems, - "nuget": ecosystemNuGet, - "actions": ecosystemActions, -} - -func mapDependencyEcosystemNaming(deps []dependency) error { - for i := range deps { - // Since we allow a dependency's ecosystem to be nil, so skip those nil ones and only map - // those valid ones. - if deps[i].Ecosystem == nil { - continue - } - mappedEcosys, err := toEcosystem(*deps[i].Ecosystem) - if err != nil { - // Iff. the ecosystem is not empty and the mapping entry is not found, we will return an error. - return fmt.Errorf("error mapping dependency ecosystem: %w", err) - } - deps[i].Ecosystem = asPointer(string(mappedEcosys)) - } - return nil -} - -// Note: the current implementation directly returns an error if the mapping entry is not found in the above hashmap. -// GitHub might update their ecosystem names frequently, so we might also need to update the above map in a timely -// manner for the dependency-diff feature not to fail because of the "mapping not found" error. -func toEcosystem(e string) (ecosystem, error) { - if ecosystemOSV, found := gitHubToOSV[e]; found { - return ecosystemOSV, nil - } - return "", fmt.Errorf("%w for github entry %s", errMappingNotFound, e) -} - -func asPointer(s string) *string { - return &s -} diff --git a/dependencydiff/raw_dependencies.go b/dependencydiff/raw_dependencies.go deleted file mode 100644 index 0d149dfc9e9..00000000000 --- a/dependencydiff/raw_dependencies.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2022 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dependencydiff - -import ( - "fmt" - "net/http" - "path" - - "github.com/google/go-github/v53/github" - - "github.com/ossf/scorecard/v5/clients/githubrepo/roundtripper" - "github.com/ossf/scorecard/v5/pkg" -) - -// dependency is a raw dependency fetched from the GitHub Dependency Review API. -// Fields of a dependency corresponding to those of pkg.DependencyCheckResult. -type dependency struct { - // Package URL is a short link for a package. - PackageURL *string `json:"package_url"` - - // SourceRepository is the source repository URL of the dependency. - SourceRepository *string `json:"source_repository_url"` - - // ChangeType indicates whether the dependency is added, updated, or removed. - ChangeType *pkg.ChangeType `json:"change_type"` - - // ManifestPath is the path of the manifest file of the dependency, such as go.mod for Go. - ManifestPath *string `json:"manifest"` - - // Ecosystem is the name of the package management system, such as NPM, GO, PYPI. - Ecosystem *string `json:"ecosystem"` - - // Version is the package version of the dependency. - Version *string `json:"version"` - - // Name is the name of the dependency. - Name string `json:"name"` -} - -// fetchRawDependencyDiffData fetches the dependency-diffs between the two code commits -// using the GitHub Dependency Review API, and returns a slice of DependencyCheckResult. -func fetchRawDependencyDiffData(dCtx *dependencydiffContext) error { - ghrt := roundtripper.NewTransport(dCtx.ctx, dCtx.logger) - ghClient := github.NewClient(&http.Client{Transport: ghrt}) - req, err := ghClient.NewRequest( - "GET", - path.Join("repos", dCtx.ownerName, dCtx.repoName, - "dependency-graph", "compare", dCtx.base+"..."+dCtx.head), - nil, - ) - if err != nil { - return fmt.Errorf("request for dependency-diff failed with %w", err) - } - _, err = ghClient.Do(dCtx.ctx, req, &dCtx.dependencydiffs) - if err != nil { - return fmt.Errorf("error parsing the dependency-diff response: %w", err) - } - for _, d := range dCtx.dependencydiffs { - if !d.ChangeType.IsValid() { - return fmt.Errorf("%w: change type", errInvalid) - } - } - return nil -} diff --git a/e2e/dependencydiff_test.go b/e2e/dependencydiff_test.go deleted file mode 100644 index e0d76461e7f..00000000000 --- a/e2e/dependencydiff_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2021 OpenSSF Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "context" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/ossf/scorecard/v5/checks" - //nolint:staticcheck // we know it's deprecated and the tests will be removed when the package is - "github.com/ossf/scorecard/v5/dependencydiff" -) - -const ( - repoURI = "ossf-tests/scorecard-depdiff" - base = "fd2a82b3b735fffbc2d782ed5f50301b879ecc51" - head = "1989568f93e484f6a86f8b276b170e3d6962ce12" -) - -var _ = Describe("E2E TEST:"+dependencydiff.Depdiff, func() { - Context("E2E TEST:Validating use of the dependency-diff API", func() { - It("Should return a slice of dependency-diff checking results", func() { - ctx := context.Background() - checksToRun := []string{ - checks.CheckBranchProtection, - } - changeTypesToCheck := []string{ - "removed", // Only checking those removed ones will make this test faster. - } - //nolint:staticcheck // we know it's deprecated and the tests will be removed when the package is - results, err := dependencydiff.GetDependencyDiffResults( - ctx, - repoURI, - base, head, - checksToRun, - changeTypesToCheck, - ) - Expect(err).Should(BeNil()) - Expect(len(results) > 0).Should(BeTrue()) - }) - It("Should return a valid empty result", func() { - ctx := context.Background() - checksToRun := []string{ - checks.CheckBranchProtection, - } - changeTypesToCheck := []string{ - "removed", - } - //nolint:staticcheck // we know it's deprecated and the tests will be removed when the package is - results, err := dependencydiff.GetDependencyDiffResults( - ctx, - repoURI, - base, base, - checksToRun, - changeTypesToCheck, - ) - Expect(err).Should(BeNil()) - Expect(len(results) == 0).Should(BeTrue()) - }) - It("Should initialize clients corresponding to the checks to run and do not crash", func() { - ctx := context.Background() - checksToRun := []string{ - checks.CheckFuzzing, - } - changeTypesToCheck := []string{ - "removed", - } - //nolint:staticcheck // we know it's deprecated and the tests will be removed when the package is - _, err := dependencydiff.GetDependencyDiffResults( - ctx, - repoURI, - base, head, - checksToRun, - changeTypesToCheck, - ) - Expect(err).Should(BeNil()) - }) - }) -}) From 290cd067932c6bd90e6a77e19157cc445ff1d7a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:14:20 +0000 Subject: [PATCH 08/15] :seedling: Bump golang.org/x/oauth2 from 0.20.0 to 0.21.0 (#4148) Signed-off-by: balteraivshay --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e330a893880..379af28d629 100644 --- a/go.mod +++ b/go.mod @@ -177,7 +177,7 @@ require ( golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.20.0 + golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index 3bb23e47a80..2cdb9e80e1d 100644 --- a/go.sum +++ b/go.sum @@ -929,8 +929,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 2d3b251135f0e6e9e7db57991c96291a969838e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:24:52 +0000 Subject: [PATCH 09/15] :seedling: Bump github.com/onsi/ginkgo/v2 in /tools (#4149) Signed-off-by: balteraivshay --- tools/go.mod | 2 +- tools/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/go.mod b/tools/go.mod index 65edc495136..3f6bdc76f09 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -8,7 +8,7 @@ require ( github.com/google/addlicense v1.1.1 github.com/google/ko v0.15.4 github.com/goreleaser/goreleaser v1.26.2 - github.com/onsi/ginkgo/v2 v2.17.3 + github.com/onsi/ginkgo/v2 v2.19.0 google.golang.org/protobuf v1.34.1 ) diff --git a/tools/go.sum b/tools/go.sum index cb96cef54e2..1eabde5dea7 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -821,8 +821,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.17.3 h1:oJcvKpIb7/8uLpDDtnQuf18xVnwKp8DTD7DQ6gTd/MU= -github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= From eeadb4327a5d6f49bdc2c6d977efff8f8c9d65fe Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Thu, 6 Jun 2024 19:42:19 +0200 Subject: [PATCH 10/15] :sparkles: announce where results are written (#4132) Before this change, when running with '-o foo' the output would end with: ``` RESULTS ------- ``` This was rather confusing. There's of course many ways to make this more clear, this commit adds a log line announcing where the output is written to: ``` RESULTS ------- Writing to foo ``` Signed-off-by: Arnout Engelen Signed-off-by: balteraivshay --- pkg/scorecard_result.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/scorecard_result.go b/pkg/scorecard_result.go index f2ae996c65e..27e9f0409ad 100644 --- a/pkg/scorecard_result.go +++ b/pkg/scorecard_result.go @@ -134,6 +134,7 @@ func FormatResults( if err != nil { return fmt.Errorf("unable to create output file: %w", err) } + fmt.Fprintln(os.Stderr, "Writing results to", opts.ResultsFile) defer output.Close() } From d01d57d4ef3fe2c4a49a2bc221c8484cdb56d419 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Thu, 6 Jun 2024 11:01:49 -0700 Subject: [PATCH 11/15] :bug: fix Unlicense detection (#4145) * fix unlicense detection The code previously had some special logic for handling the Unlicense SPDX identifier. While this worked for local file detection, it broke detection for SPDX identifiers provided by the forge. This change moves the logic to the part of the code concerned with local file detection, so both work now. Signed-off-by: Spencer Schrock * remove part of comment which is no longer relevant Signed-off-by: Spencer Schrock --------- Signed-off-by: Spencer Schrock Signed-off-by: balteraivshay --- checks/raw/license.go | 43 +++++++++++++++++-------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/checks/raw/license.go b/checks/raw/license.go index 3e041c7e2a4..af0f83d92fb 100644 --- a/checks/raw/license.go +++ b/checks/raw/license.go @@ -160,17 +160,10 @@ func License(c *checker.CheckRequest) (checker.LicenseData, error) { // scorecard search stops at first candidate (isLicenseFile) license file found if path != (checker.LicenseFile{}) { - // - // now it is time to "map it back" in the case of the - // Spdx Identifier for "UNLICENSE" which was mapped to "UN" - // for the regex group match and this check. - // grab what is needed before clobbering the Spdx Identifier - // Aside from 'UN', these settings (Name, Key) match GH repo API - // for when the Spdx Identifier cannot be determined. path.LicenseInformation.Name = fsfOsiApprovedLicenseCiMap[strings.ToUpper(path.LicenseInformation.SpdxID)].Name - if strings.ToUpper(path.LicenseInformation.SpdxID) == "UN" { - path.LicenseInformation.SpdxID = "UNLICENSE" - } else if path.LicenseInformation.SpdxID == "" { + // these settings (Name, Key) match GH repo API + // for when the Spdx Identifier cannot be determined. + if path.LicenseInformation.SpdxID == "" { path.LicenseInformation.SpdxID = "NOASSERTION" path.LicenseInformation.Name = "Other" } @@ -226,19 +219,7 @@ func setCiMap() { defer ciMapMutex.Unlock() if len(fsfOsiApprovedLicenseCiMap) == 0 { for key, entry := range fsfOsiApprovedLicenseMap { - // Special case, the unlicense, in the map is - // called 'The Unlicense' with the Spdx id 'Unlicense'. - // For the regex's 'un' will match the [pre|suf]Spdx - // regex group (just as it would match '0BSD'), but - // 'un' will not "hit" in the map with key 'Unlicense' - // so change to 'UN' for 'unlicense' for 'isLicenseFile()' - // TODO: make this general (pass a key map for changing these - // special cases). For now this is the only one. - if strings.ToUpper(key) == "UNLICENSE" { - fsfOsiApprovedLicenseCiMap["UN"] = entry - } else { - fsfOsiApprovedLicenseCiMap[strings.ToUpper(key)] = entry - } + fsfOsiApprovedLicenseCiMap[strings.ToUpper(key)] = entry } } } @@ -261,12 +242,22 @@ func getSpdxID(matches []string) string { // value, preSpdx takes precedence. // (e.g., 0BSD-LICENSE-GPL-2.0.txt) // TODO: decide if that is OK or should "fail" + var id string if matches[reGroupIdxs["preSpdx"]] != "" { - return matches[reGroupIdxs["preSpdx"]] + id = matches[reGroupIdxs["preSpdx"]] } else if matches[reGroupIdxs["sufSpdx"]] != "" { - return matches[reGroupIdxs["sufSpdx"]] + id = matches[reGroupIdxs["sufSpdx"]] } - return "" + // Special case, the unlicense, in the map is + // called 'The Unlicense' with the Spdx id 'Unlicense'. + // For the regex's 'un' will match the [pre|suf]Spdx + // regex group (just as it would match '0BSD'), but + // 'un' will not "hit" in the map with key 'Unlicense' + if strings.EqualFold(id, "UN") { + id = "UNLICENSE" + } + + return id } func getExt(filename string, matches []string) string { From a609128a0404b3291f306856e8e822f0f74016c0 Mon Sep 17 00:00:00 2001 From: balteraivshay Date: Fri, 7 Jun 2024 15:51:16 +0000 Subject: [PATCH 12/15] fix lint Signed-off-by: balteraivshay --- checks/raw/shell_download_validate.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/checks/raw/shell_download_validate.go b/checks/raw/shell_download_validate.go index 8f59aba04e5..488dbe88b0f 100644 --- a/checks/raw/shell_download_validate.go +++ b/checks/raw/shell_download_validate.go @@ -790,7 +790,11 @@ func isUnpinnedDotNetCliAdd(cmd []string) bool { } func isNuget(cmd []string) bool { - return isDotNetCliAdd(cmd) || isNugetCliInstall(cmd) || isDotNetCliRestore(cmd) || isNugetCliRestore(cmd) || isMsBuildRestore(cmd) + return isDotNetCliAdd(cmd) || + isNugetCliInstall(cmd) || + isDotNetCliRestore(cmd) || + isNugetCliRestore(cmd) || + isMsBuildRestore(cmd) } func isNugetUnpinned(cmd []string) bool { @@ -854,10 +858,6 @@ func isMsBuildRestore(cmd []string) bool { return false } -func isNugetRestore(cmd []string) bool { - return isDotNetCliRestore(cmd) || isNugetCliRestore(cmd) || isMsBuildRestore(cmd) -} - func isUnpinnedNugetCliRestore(cmd []string) bool { unpinnedDependency := true for i := 2; i < len(cmd); i++ { From 850182450aa8d66854d463ed376a3316a2fbbc75 Mon Sep 17 00:00:00 2001 From: Raghav Kaul <8695110+raghavkaul@users.noreply.github.com> Date: Fri, 7 Jun 2024 10:15:20 -0700 Subject: [PATCH 13/15] =?UTF-8?q?=E2=9C=A8=20probe:=20releases=20with=20ve?= =?UTF-8?q?rified=20provenance=20(#4141)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add projectpackageversions to signed releases raw results Signed-off-by: Raghav Kaul * finding: add NewNot* helpers, fix error msg Signed-off-by: Raghav Kaul * probe: releasesHaveVerifiedProvenance Signed-off-by: Raghav Kaul * logging Signed-off-by: Raghav Kaul * fix tests and lint Signed-off-by: Raghav Kaul * address comments Signed-off-by: Raghav Kaul * remove unused Signed-off-by: Raghav Kaul * fix merge conflict Signed-off-by: Raghav Kaul --------- Signed-off-by: Raghav Kaul Signed-off-by: balteraivshay --- checks/evaluation/signed_releases.go | 9 ++ finding/finding.go | 12 ++ finding/probe.go | 2 +- options/flags.go | 1 + probes/entries.go | 2 + probes/releasesHaveVerifiedProvenance/def.yml | 34 +++++ probes/releasesHaveVerifiedProvenance/impl.go | 70 ++++++++++ .../impl_test.go | 121 ++++++++++++++++++ 8 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 probes/releasesHaveVerifiedProvenance/def.yml create mode 100644 probes/releasesHaveVerifiedProvenance/impl.go create mode 100644 probes/releasesHaveVerifiedProvenance/impl_test.go diff --git a/checks/evaluation/signed_releases.go b/checks/evaluation/signed_releases.go index f432556beb5..1fc9cfb0524 100644 --- a/checks/evaluation/signed_releases.go +++ b/checks/evaluation/signed_releases.go @@ -24,6 +24,7 @@ import ( "github.com/ossf/scorecard/v5/finding" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" + "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" ) var errNoReleaseFound = errors.New("no release found") @@ -55,6 +56,10 @@ func SignedReleases(name string, for i := range findings { f := &findings[i] + if f.Probe == releasesHaveVerifiedProvenance.Probe { + continue + } + // Debug release name if f.Outcome == finding.OutcomeNotApplicable { // Generic summary. @@ -86,6 +91,10 @@ func SignedReleases(name string, for i := range findings { f := &findings[i] + if f.Probe == releasesHaveVerifiedProvenance.Probe { + continue + } + releaseName := getReleaseName(f) if releaseName == "" { return checker.CreateRuntimeErrorResult(name, errNoReleaseFound) diff --git a/finding/finding.go b/finding/finding.go index ad1da29cc1f..6745f8a0ce1 100644 --- a/finding/finding.go +++ b/finding/finding.go @@ -144,12 +144,24 @@ func NewFalse(efs embed.FS, probeID, text string, loc *Location, return NewWith(efs, probeID, text, loc, OutcomeFalse) } +// NewNotApplicable create a finding with a NotApplicable outcome and the desired location. +func NewNotApplicable(efs embed.FS, probeID, text string, loc *Location, +) (*Finding, error) { + return NewWith(efs, probeID, text, loc, OutcomeNotApplicable) +} + // NewNotAvailable create a finding with a NotAvailable outcome and the desired location. func NewNotAvailable(efs embed.FS, probeID, text string, loc *Location, ) (*Finding, error) { return NewWith(efs, probeID, text, loc, OutcomeNotAvailable) } +// NewNotSupported create a finding with a NotSupported outcome and the desired location. +func NewNotSupported(efs embed.FS, probeID, text string, loc *Location, +) (*Finding, error) { + return NewWith(efs, probeID, text, loc, OutcomeNotSupported) +} + // NewTrue create a true finding with the desired location. func NewTrue(efs embed.FS, probeID, text string, loc *Location, ) (*Finding, error) { diff --git a/finding/probe.go b/finding/probe.go index ec6ef66c83b..0acf821e561 100644 --- a/finding/probe.go +++ b/finding/probe.go @@ -142,7 +142,7 @@ func validateID(actual, expected string) error { func validateRemediation(r yamlRemediation) error { if err := validateRemediationOutcomeTrigger(r.OnOutcome); err != nil { - return err + return fmt.Errorf("remediation: %w", err) } switch r.Effort { case RemediationEffortHigh, RemediationEffortMedium, RemediationEffortLow: diff --git a/options/flags.go b/options/flags.go index 07eaeae21c1..dba6fcc2611 100644 --- a/options/flags.go +++ b/options/flags.go @@ -193,6 +193,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) { allowedFormats := []string{ FormatDefault, FormatJSON, + FormatProbe, } if o.isSarifEnabled() { diff --git a/probes/entries.go b/probes/entries.go index 527bb485898..4ec8941188c 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -48,6 +48,7 @@ import ( "github.com/ossf/scorecard/v5/probes/pinsDependencies" "github.com/ossf/scorecard/v5/probes/releasesAreSigned" "github.com/ossf/scorecard/v5/probes/releasesHaveProvenance" + "github.com/ossf/scorecard/v5/probes/releasesHaveVerifiedProvenance" "github.com/ossf/scorecard/v5/probes/requiresApproversForPullRequests" "github.com/ossf/scorecard/v5/probes/requiresCodeOwnersReview" "github.com/ossf/scorecard/v5/probes/requiresLastPushApproval" @@ -168,6 +169,7 @@ var ( hasPermissiveLicense.Run, codeReviewOneReviewers.Run, hasBinaryArtifacts.Run, + releasesHaveVerifiedProvenance.Run, } // Probes which don't use pre-computed raw data but rather collect it themselves. diff --git a/probes/releasesHaveVerifiedProvenance/def.yml b/probes/releasesHaveVerifiedProvenance/def.yml new file mode 100644 index 00000000000..698c520bf3d --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/def.yml @@ -0,0 +1,34 @@ +# Copyright 2024 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: releasesHaveVerifiedProvenance +short: Checks if the project releases with provenance attestations that have been verified +motivation: > + Package provenance attestations provide a greater guarantee of authenticity and integrity than package signatures alone, since the attestation can be performed over a hash of both the package contents and metadata. Developers can attest to particular qualities of the build, such as the build environment, build steps or builder identity. +implementation: > + This probe checks how many packages published by the repository are associated with verified SLSA provenance attestations. It uses data from a ProjectPackageClient, which associates a GitHub/GitLab project with a package in a package manager. Using the data from the package manager (whom we rely on to verify the provenance attestation), this probe returns a finding for each release. For now, only NPM is supported. +outcome: + - For each release, the probe returns OutcomeTrue or OutcomeFalse, depending on if the package has a verified provenance attestation. + - If we didn't find a package or didn't find releases, return OutcomeNotAvailable. +remediation: + onOutcome: False + effort: Low + text: + - For NPM, publish provenance alongside your package using the `--provenance` flag (See (Introducing npm package provenance)[https://github.blog/2023-04-19-introducing-npm-package-provenance/]) +ecosystem: + languages: + - javascript + clients: + - github + - gitlab diff --git a/probes/releasesHaveVerifiedProvenance/impl.go b/probes/releasesHaveVerifiedProvenance/impl.go new file mode 100644 index 00000000000..f73b20fcc56 --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/impl.go @@ -0,0 +1,70 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveVerifiedProvenance + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" + "github.com/ossf/scorecard/v5/internal/probes" +) + +func init() { + probes.MustRegister(Probe, Run, []probes.CheckName{probes.SignedReleases}) +} + +//go:embed *.yml +var fs embed.FS + +const ( + Probe = "releasesHaveVerifiedProvenance" +) + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + var findings []finding.Finding + + if len(raw.SignedReleasesResults.Packages) == 0 { + f, err := finding.NewNotApplicable(fs, Probe, "no package manager releases found", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + return findings, Probe, nil + } + + for i := range raw.SignedReleasesResults.Packages { + p := raw.SignedReleasesResults.Packages[i] + + if !p.Provenance.IsVerified { + f, err := finding.NewFalse(fs, Probe, "release without verified provenance", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + continue + } + + f, err := finding.NewTrue(fs, Probe, "release with verified provenance", nil) + if err != nil { + return []finding.Finding{}, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + + return findings, Probe, nil +} diff --git a/probes/releasesHaveVerifiedProvenance/impl_test.go b/probes/releasesHaveVerifiedProvenance/impl_test.go new file mode 100644 index 00000000000..317638b5b8a --- /dev/null +++ b/probes/releasesHaveVerifiedProvenance/impl_test.go @@ -0,0 +1,121 @@ +// Copyright 2024 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveVerifiedProvenance + +import ( + "errors" + "testing" + + "github.com/ossf/scorecard/v5/checker" + "github.com/ossf/scorecard/v5/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + //nolint:govet + tests := []struct { + desc string + pkgs []checker.ProjectPackage + outcomes []finding.Outcome + err error + }{ + { + desc: "no packages found", + outcomes: []finding.Outcome{finding.OutcomeNotApplicable}, + }, + { + desc: "some releases with verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + { + Name: "a", + Version: "1.0.1", + }, + }, + outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeFalse}, + }, + { + desc: "all releases with verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + { + Name: "a", + Version: "1.0.1", + Provenance: checker.PackageProvenance{IsVerified: true}, + }, + }, + outcomes: []finding.Outcome{finding.OutcomeTrue, finding.OutcomeTrue}, + }, + { + desc: "no verified provenance", + pkgs: []checker.ProjectPackage{ + { + Name: "a", + Version: "1.0.0", + }, + { + Name: "a", + Version: "1.0.1", + }, + }, + outcomes: []finding.Outcome{finding.OutcomeFalse, finding.OutcomeFalse}, + }, + } + + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.desc, func(t *testing.T) { + t.Parallel() + raw := checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Packages: tt.pkgs, + }, + } + + outcomes, _, err := Run(&raw) + + if !errors.Is(tt.err, err) { + t.Errorf("expected %+v got %+v", tt.err, err) + } + + if !cmpOutcomes(tt.outcomes, outcomes) { + t.Errorf("expected %+v got %+v", tt.outcomes, outcomes) + } + }) + } +} + +func cmpOutcomes(ex []finding.Outcome, act []finding.Finding) bool { + if len(ex) != len(act) { + return false + } + + for i := range ex { + if act[i].Outcome != ex[i] { + return false + } + } + + return true +} From 9e66eb2280d5f7f57768d610841f3ca5c9df0141 Mon Sep 17 00:00:00 2001 From: balteraivshay Date: Wed, 12 Jun 2024 08:36:06 +0000 Subject: [PATCH 14/15] fix shell download Signed-off-by: balteraivshay --- checks/raw/shell_download_validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/raw/shell_download_validate.go b/checks/raw/shell_download_validate.go index 488dbe88b0f..c8e5d1d8a16 100644 --- a/checks/raw/shell_download_validate.go +++ b/checks/raw/shell_download_validate.go @@ -36,7 +36,7 @@ import ( var ( // supportedShells is the list of shells that are supported by mvdan.cc/sh/v3/syntax. supportedShells = []string{ - "sh", "bash", "mksh", + "sh", "bash", "mksh", "powershell", } // otherShells are not supported by our parser. otherShells = []string{ From dde4f2482aac1d674dde734b5aa4baf226e72b88 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Wed, 10 Jul 2024 15:39:17 -0700 Subject: [PATCH 15/15] Revert "fix shell download" This reverts commit 9e66eb2280d5f7f57768d610841f3ca5c9df0141. Signed-off-by: Spencer Schrock --- checks/raw/shell_download_validate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checks/raw/shell_download_validate.go b/checks/raw/shell_download_validate.go index c8e5d1d8a16..488dbe88b0f 100644 --- a/checks/raw/shell_download_validate.go +++ b/checks/raw/shell_download_validate.go @@ -36,7 +36,7 @@ import ( var ( // supportedShells is the list of shells that are supported by mvdan.cc/sh/v3/syntax. supportedShells = []string{ - "sh", "bash", "mksh", "powershell", + "sh", "bash", "mksh", } // otherShells are not supported by our parser. otherShells = []string{