From de28e92cbede89f79ced31457d7e326da1f2b35c Mon Sep 17 00:00:00 2001 From: Chuang Wang Date: Thu, 24 Aug 2023 18:00:01 -0400 Subject: [PATCH] Deep inspection for materials (#903) Step 2/2 of https://github.com/tektoncd/chains/issues/850 Add deep inspection for materials, which will be applied to both slsa v0.2 and slsa v1.0 provenance. Signed-off-by: Chuang Wang --- .../slsa/internal/material/material.go | 31 ++++-- .../slsa/internal/material/material_test.go | 100 +++++++++++++++--- .../slsa/v1/pipelinerun/pipelinerun.go | 2 +- .../internal/pipelinerun/pipelinerun.go | 2 +- .../resolved_dependencies.go | 7 +- .../resolved_dependencies_test.go | 14 ++- 6 files changed, 126 insertions(+), 30 deletions(-) diff --git a/pkg/chains/formats/slsa/internal/material/material.go b/pkg/chains/formats/slsa/internal/material/material.go index f24f5a6589..c61df507e3 100644 --- a/pkg/chains/formats/slsa/internal/material/material.go +++ b/pkg/chains/formats/slsa/internal/material/material.go @@ -26,6 +26,7 @@ import ( "github.com/tektoncd/chains/internal/backport" "github.com/tektoncd/chains/pkg/artifacts" "github.com/tektoncd/chains/pkg/chains/formats/slsa/attest" + "github.com/tektoncd/chains/pkg/chains/formats/slsa/internal/slsaconfig" "github.com/tektoncd/chains/pkg/chains/objects" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" "knative.dev/pkg/logging" @@ -67,7 +68,7 @@ func TaskMaterials(ctx context.Context, tro *objects.TaskRunObject) ([]common.Pr return mats, nil } -func PipelineMaterials(ctx context.Context, pro *objects.PipelineRunObject) ([]common.ProvenanceMaterial, error) { +func PipelineMaterials(ctx context.Context, pro *objects.PipelineRunObject, slsaconfig *slsaconfig.SlsaConfig) ([]common.ProvenanceMaterial, error) { logger := logging.FromContext(ctx) var mats []common.ProvenanceMaterial if p := pro.Status.Provenance; p != nil && p.RefSource != nil { @@ -112,7 +113,7 @@ func PipelineMaterials(ctx context.Context, pro *objects.PipelineRunObject) ([]c } } - mats = append(mats, FromPipelineParamsAndResults(ctx, pro)...) + mats = append(mats, FromPipelineParamsAndResults(ctx, pro, slsaconfig)...) // remove duplicate materials mats, err := removeDuplicateMaterials(mats) @@ -289,15 +290,33 @@ func removeDuplicateMaterials(mats []common.ProvenanceMaterial) ([]common.Proven } // FromPipelineParamsAndResults extracts type hinted params and results and adds the url and digest to materials. -func FromPipelineParamsAndResults(ctx context.Context, pro *objects.PipelineRunObject) []common.ProvenanceMaterial { +func FromPipelineParamsAndResults(ctx context.Context, pro *objects.PipelineRunObject, slsaconfig *slsaconfig.SlsaConfig) []common.ProvenanceMaterial { mats := []common.ProvenanceMaterial{} sms := artifacts.RetrieveMaterialsFromStructuredResults(ctx, pro, artifacts.ArtifactsInputsResultName) mats = append(mats, sms...) var commit, url string - // search status.PipelineSpec.params - if pro.Status.PipelineSpec != nil { - for _, p := range pro.Status.PipelineSpec.Params { + + pSpec := pro.Status.PipelineSpec + if pSpec != nil { + // search type hinting param/results from each individual taskruns + if slsaconfig.DeepInspectionEnabled { + logger := logging.FromContext(ctx) + pipelineTasks := append(pSpec.Tasks, pSpec.Finally...) + for _, t := range pipelineTasks { + tr := pro.GetTaskRunFromTask(t.Name) + // Ignore Tasks that did not execute during the PipelineRun. + if tr == nil || tr.Status.CompletionTime == nil { + logger.Infof("taskrun is not found or not completed for the task %s", t.Name) + continue + } + materialsFromTasks := FromTaskParamsAndResults(ctx, objects.NewTaskRunObject(tr)) + mats = append(mats, materialsFromTasks...) + } + } + + // search status.PipelineSpec.params + for _, p := range pSpec.Params { if p.Default == nil { continue } diff --git a/pkg/chains/formats/slsa/internal/material/material_test.go b/pkg/chains/formats/slsa/internal/material/material_test.go index cdd2073011..6b19c1d26c 100644 --- a/pkg/chains/formats/slsa/internal/material/material_test.go +++ b/pkg/chains/formats/slsa/internal/material/material_test.go @@ -21,16 +21,19 @@ import ( "reflect" "strings" "testing" + "time" "github.com/google/go-cmp/cmp" "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/common" "github.com/tektoncd/chains/internal/backport" "github.com/tektoncd/chains/pkg/artifacts" "github.com/tektoncd/chains/pkg/chains/formats/slsa/internal/compare" + "github.com/tektoncd/chains/pkg/chains/formats/slsa/internal/slsaconfig" "github.com/tektoncd/chains/pkg/chains/objects" "github.com/tektoncd/chains/pkg/internal/objectloader" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" "github.com/tektoncd/pipeline/pkg/apis/resource/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" logtesting "knative.dev/pkg/logging/testing" "sigs.k8s.io/yaml" ) @@ -357,7 +360,7 @@ func TestPipelineMaterials(t *testing.T) { {URI: artifacts.GitSchemePrefix + "https://git.test.com.git", Digest: common.DigestSet{"sha1": "abcd"}}, } ctx := logtesting.TestContextWithLogger(t) - got, err := PipelineMaterials(ctx, createPro("../../testdata/pipelinerun1.json")) + got, err := PipelineMaterials(ctx, createPro("../../testdata/pipelinerun1.json"), &slsaconfig.SlsaConfig{DeepInspectionEnabled: false}) if err != nil { t.Error(err) } @@ -391,7 +394,7 @@ func TestStructuredResultPipelineMaterials(t *testing.T) { }, } ctx := logtesting.TestContextWithLogger(t) - got, err := PipelineMaterials(ctx, createPro("../../testdata/pipelinerun_structured_results.json")) + got, err := PipelineMaterials(ctx, createPro("../../testdata/pipelinerun_structured_results.json"), &slsaconfig.SlsaConfig{DeepInspectionEnabled: false}) if err != nil { t.Errorf("error while extracting materials: %v", err) } @@ -712,14 +715,16 @@ func TestRemoveDuplicates(t *testing.T) { } } +//nolint:all func TestFromPipelineParamsAndResults(t *testing.T) { tests := []struct { - name string - pipelineRun *v1beta1.PipelineRun - want []common.ProvenanceMaterial + name string + pipelineRunObject *objects.PipelineRunObject + enableDeepInspection bool + want []common.ProvenanceMaterial }{{ name: "from results", - pipelineRun: &v1beta1.PipelineRun{ + pipelineRunObject: objects.NewPipelineRunObject(&v1beta1.PipelineRun{ Status: v1beta1.PipelineRunStatus{ PipelineRunStatusFields: v1beta1.PipelineRunStatusFields{ PipelineResults: []v1beta1.PipelineRunResult{{ @@ -731,7 +736,7 @@ func TestFromPipelineParamsAndResults(t *testing.T) { }}, }, }, - }, + }), want: []common.ProvenanceMaterial{{ URI: "git+github.com/something.git", Digest: common.DigestSet{ @@ -740,7 +745,7 @@ func TestFromPipelineParamsAndResults(t *testing.T) { }}, }, { name: "from pipelinespec", - pipelineRun: &v1beta1.PipelineRun{ + pipelineRunObject: objects.NewPipelineRunObject(&v1beta1.PipelineRun{ Status: v1beta1.PipelineRunStatus{ PipelineRunStatusFields: v1beta1.PipelineRunStatusFields{ PipelineSpec: &v1beta1.PipelineSpec{ @@ -758,7 +763,7 @@ func TestFromPipelineParamsAndResults(t *testing.T) { }, }, }, - }, + }), want: []common.ProvenanceMaterial{{ URI: "git+github.com/something.git", Digest: common.DigestSet{ @@ -767,7 +772,7 @@ func TestFromPipelineParamsAndResults(t *testing.T) { }}, }, { name: "from pipelineRunSpec", - pipelineRun: &v1beta1.PipelineRun{ + pipelineRunObject: objects.NewPipelineRunObject(&v1beta1.PipelineRun{ Spec: v1beta1.PipelineRunSpec{ Params: []v1beta1.Param{{ Name: "CHAINS-GIT_COMMIT", @@ -781,7 +786,7 @@ func TestFromPipelineParamsAndResults(t *testing.T) { }, }}, }, - }, + }), want: []common.ProvenanceMaterial{{ URI: "git+github.com/something.git", Digest: common.DigestSet{ @@ -790,7 +795,7 @@ func TestFromPipelineParamsAndResults(t *testing.T) { }}, }, { name: "from completeChain", - pipelineRun: &v1beta1.PipelineRun{ + pipelineRunObject: objects.NewPipelineRunObject(&v1beta1.PipelineRun{ Spec: v1beta1.PipelineRunSpec{ Params: []v1beta1.Param{{ Name: "CHAINS-GIT_URL", @@ -812,21 +817,84 @@ func TestFromPipelineParamsAndResults(t *testing.T) { }}, }, }, - }, + }), want: []common.ProvenanceMaterial{{ URI: "git+github.com/something.git", Digest: common.DigestSet{ "sha1": "my-commit", }, }}, - }} + }, { + name: "deep inspection: pipelinerun param and task result", + pipelineRunObject: createProWithPipelineParamAndTaskResult(), + enableDeepInspection: true, + want: []common.ProvenanceMaterial{ + { + URI: "git+github.com/pipelinerun-param.git", + Digest: common.DigestSet{ + "sha1": "115734d92807a80158b4b7af605d768c647fdb3d", + }, + }, { + URI: "github.com/childtask-result", + Digest: common.DigestSet{ + "sha1": "225734d92807a80158b4b7af605d768c647fdb3d", + }, + }, + }, + }, + } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { ctx := logtesting.TestContextWithLogger(t) - got := FromPipelineParamsAndResults(ctx, objects.NewPipelineRunObject(tc.pipelineRun)) - if diff := cmp.Diff(tc.want, got); diff != "" { + got := FromPipelineParamsAndResults(ctx, tc.pipelineRunObject, &slsaconfig.SlsaConfig{DeepInspectionEnabled: tc.enableDeepInspection}) + if diff := cmp.Diff(tc.want, got, compare.MaterialsCompareOption()); diff != "" { t.Errorf("FromPipelineParamsAndResults(): -want +got: %s", diff) } }) } } + +//nolint:all +func createProWithPipelineParamAndTaskResult() *objects.PipelineRunObject { + pro := objects.NewPipelineRunObject(&v1beta1.PipelineRun{ + Status: v1beta1.PipelineRunStatus{ + PipelineRunStatusFields: v1beta1.PipelineRunStatusFields{ + PipelineSpec: &v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "CHAINS-GIT_COMMIT", + Default: &v1beta1.ParamValue{ + StringVal: "115734d92807a80158b4b7af605d768c647fdb3d", + }, + }, { + Name: "CHAINS-GIT_URL", + Default: &v1beta1.ParamValue{ + StringVal: "github.com/pipelinerun-param", + }, + }}, + }, + }, + }, + }) + + pipelineTaskName := "my-clone-task" + tr := &v1beta1.TaskRun{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{objects.PipelineTaskLabel: pipelineTaskName}}, + Status: v1beta1.TaskRunStatus{ + TaskRunStatusFields: v1beta1.TaskRunStatusFields{ + CompletionTime: &metav1.Time{Time: time.Date(1995, time.December, 24, 6, 12, 12, 24, time.UTC)}, + TaskRunResults: []v1beta1.TaskRunResult{ + { + Name: "ARTIFACT_INPUTS", + Value: *v1beta1.NewObject(map[string]string{ + "uri": "github.com/childtask-result", + "digest": "sha1:225734d92807a80158b4b7af605d768c647fdb3d", + })}, + }, + }, + }, + } + + pro.AppendTaskRun(tr) + pro.Status.PipelineSpec.Tasks = []v1beta1.PipelineTask{{Name: pipelineTaskName}} + return pro +} diff --git a/pkg/chains/formats/slsa/v1/pipelinerun/pipelinerun.go b/pkg/chains/formats/slsa/v1/pipelinerun/pipelinerun.go index 984450e8f9..89f65fab17 100644 --- a/pkg/chains/formats/slsa/v1/pipelinerun/pipelinerun.go +++ b/pkg/chains/formats/slsa/v1/pipelinerun/pipelinerun.go @@ -50,7 +50,7 @@ type TaskAttestation struct { func GenerateAttestation(ctx context.Context, pro *objects.PipelineRunObject, slsaConfig *slsaconfig.SlsaConfig) (interface{}, error) { subjects := extract.SubjectDigests(ctx, pro, slsaConfig) - mat, err := material.PipelineMaterials(ctx, pro) + mat, err := material.PipelineMaterials(ctx, pro, slsaConfig) if err != nil { return nil, err } diff --git a/pkg/chains/formats/slsa/v2alpha2/internal/pipelinerun/pipelinerun.go b/pkg/chains/formats/slsa/v2alpha2/internal/pipelinerun/pipelinerun.go index f9f334a44e..56ff0c311e 100644 --- a/pkg/chains/formats/slsa/v2alpha2/internal/pipelinerun/pipelinerun.go +++ b/pkg/chains/formats/slsa/v2alpha2/internal/pipelinerun/pipelinerun.go @@ -34,7 +34,7 @@ const ( // GenerateAttestation generates a provenance statement with SLSA v1.0 predicate for a pipeline run. func GenerateAttestation(ctx context.Context, pro *objects.PipelineRunObject, slsaconfig *slsaconfig.SlsaConfig) (interface{}, error) { - rd, err := resolveddependencies.PipelineRun(ctx, pro) + rd, err := resolveddependencies.PipelineRun(ctx, pro, slsaconfig) if err != nil { return nil, err } diff --git a/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies.go b/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies.go index 19abb04bd9..6bc1436a67 100644 --- a/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies.go +++ b/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies.go @@ -23,6 +23,7 @@ import ( "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/common" v1 "github.com/in-toto/in-toto-golang/in_toto/slsa_provenance/v1" "github.com/tektoncd/chains/pkg/chains/formats/slsa/internal/material" + "github.com/tektoncd/chains/pkg/chains/formats/slsa/internal/slsaconfig" "github.com/tektoncd/chains/pkg/chains/objects" "go.uber.org/zap" "knative.dev/pkg/logging" @@ -89,7 +90,7 @@ func TaskRun(ctx context.Context, tro *objects.TaskRunObject) ([]v1.ResourceDesc } // PipelineRun constructs `predicate.resolvedDependencies` section by collecting all the artifacts that influence a pipeline run such as source code repo and step&sidecar base images. -func PipelineRun(ctx context.Context, pro *objects.PipelineRunObject) ([]v1.ResourceDescriptor, error) { +func PipelineRun(ctx context.Context, pro *objects.PipelineRunObject, slsaconfig *slsaconfig.SlsaConfig) ([]v1.ResourceDescriptor, error) { var err error var resolvedDependencies []v1.ResourceDescriptor logger := logging.FromContext(ctx) @@ -112,7 +113,7 @@ func PipelineRun(ctx context.Context, pro *objects.PipelineRunObject) ([]v1.Reso resolvedDependencies = append(resolvedDependencies, rds...) // add resolved dependencies from pipeline results - mats := material.FromPipelineParamsAndResults(ctx, pro) + mats := material.FromPipelineParamsAndResults(ctx, pro, slsaconfig) // convert materials to resolved dependencies resolvedDependencies = append(resolvedDependencies, convertMaterialsToResolvedDependencies(mats, inputResultName)...) @@ -147,7 +148,7 @@ func removeDuplicateResolvedDependencies(resolvedDependencies []v1.ResourceDescr // make map to store seen resolved dependencies seen := map[string]bool{} for _, resolvedDependency := range resolvedDependencies { - // Since resolvedDependencies contain names, we want to igmore those while checking for duplicates. + // Since resolvedDependencies contain names, we want to ignore those while checking for duplicates. // Therefore, make a copy of the resolved dependency that only contains the uri and digest fields. rDep := v1.ResourceDescriptor{} rDep.URI = resolvedDependency.URI diff --git a/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies_test.go b/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies_test.go index fb0b949cab..046d2fc102 100644 --- a/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies_test.go +++ b/pkg/chains/formats/slsa/v2alpha2/internal/resolved_dependencies/resolved_dependencies_test.go @@ -26,6 +26,7 @@ import ( "github.com/tektoncd/chains/internal/backport" "github.com/tektoncd/chains/pkg/artifacts" "github.com/tektoncd/chains/pkg/chains/formats/slsa/internal/compare" + "github.com/tektoncd/chains/pkg/chains/formats/slsa/internal/slsaconfig" "github.com/tektoncd/chains/pkg/chains/objects" "github.com/tektoncd/chains/pkg/internal/objectloader" "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1" @@ -504,7 +505,7 @@ func TestPipelineRun(t *testing.T) { {Name: "inputs/result", URI: "git+https://git.test.com.git", Digest: common.DigestSet{"sha1": "abcd"}}, } ctx := logtesting.TestContextWithLogger(t) - got, err := PipelineRun(ctx, pro) + got, err := PipelineRun(ctx, pro, &slsaconfig.SlsaConfig{DeepInspectionEnabled: false}) if err != nil { t.Error(err) } @@ -532,14 +533,21 @@ func TestPipelineRunStructuredResult(t *testing.T) { }, { Name: "inputs/result", - URI: "abcd", + URI: "abc", Digest: common.DigestSet{ "sha256": "827521c857fdcd4374f4da5442fbae2edb01e7fbae285c3ec15673d4c1daecb7", }, }, + { + Name: "inputs/result", + URI: "git+https://git.test.com.git", + Digest: common.DigestSet{ + "sha1": "abcd", + }, + }, } ctx := logtesting.TestContextWithLogger(t) - got, err := PipelineRun(ctx, proStructuredResults) + got, err := PipelineRun(ctx, pro, &slsaconfig.SlsaConfig{DeepInspectionEnabled: false}) if err != nil { t.Errorf("error while extracting resolvedDependencies: %v", err) }