From 900701c045a54a634f747dc6b3a78f53b3985f4e Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Wed, 19 Oct 2022 11:41:42 -0700 Subject: [PATCH 1/3] Fix typo which prevented metadata from going to cron job. (#2370) Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock --- cron/internal/worker/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/internal/worker/main.go b/cron/internal/worker/main.go index d2637d1eb4c..fe6782a1019 100644 --- a/cron/internal/worker/main.go +++ b/cron/internal/worker/main.go @@ -91,7 +91,7 @@ func processRequest(ctx context.Context, logger.Info(fmt.Sprintf("invalid GitHub URL: %v", err)) continue } - repo.AppendMetadata(repo.Metadata()...) + repo.AppendMetadata(repoReq.Metadata...) commitSHA := clients.HeadSHA requiredRequestType := []checker.RequestType{} From f979097a1fd767718f9d05489cda6251486ca74c Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Wed, 19 Oct 2022 14:01:42 -0700 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=8C=B1=20cron:=20generalize=20and=20e?= =?UTF-8?q?xpose=20worker=20(6/n)=20(#2317)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP Signed-off-by: Spencer Schrock * Appease linter. Signed-off-by: Spencer Schrock * Update Makefile for worker Signed-off-by: Spencer Schrock * Extract already completed request sanity check. Signed-off-by: Spencer Schrock * Add worker test. Signed-off-by: Spencer Schrock * remove logger from worker interface Signed-off-by: Spencer Schrock * move cron data and worker out of cron/internal Signed-off-by: Spencer Schrock * Move config out of internal. Signed-off-by: Spencer Schrock * Document worker interface. Signed-off-by: Spencer Schrock * Fix typo which prevented metadata from going to cron job. Signed-off-by: Spencer Schrock * Address feedback. Signed-off-by: Spencer Schrock * Revert "Fix typo which prevented metadata from going to cron job." This reverts commit 876acb062e8a562cb9ed0d2e3e2a76c2babd850b. Will send separate PR. Signed-off-by: Spencer Schrock * Fix linter. Signed-off-by: Spencer Schrock Signed-off-by: Spencer Schrock --- Makefile | 22 +- cron/{internal => }/config/config.go | 0 cron/{internal => }/config/config.yaml | 0 cron/{internal => }/config/config_test.go | 0 .../{internal => }/config/testdata/basic.yaml | 0 .../config/testdata/missing_field.yaml | 0 .../config/testdata/optional_maps.yaml | 0 cron/{internal => }/data/README.md | 0 cron/{internal => }/data/blob.go | 2 +- cron/{internal => }/data/blob_test.go | 0 cron/{internal => }/data/format.go | 0 cron/{internal => }/data/format_test.go | 0 cron/{internal => }/data/iterator.go | 0 cron/{internal => }/data/iterator_test.go | 0 cron/data/metadata.pb.go | 185 +++++++++++++++ cron/{internal => }/data/metadata.proto | 2 +- cron/{internal => }/data/request.pb.go | 124 +++++----- cron/{internal => }/data/request.proto | 2 +- cron/{internal => }/data/testdata/basic.csv | 0 .../data/testdata/blob_test/key1.txt | 0 .../data/testdata/blob_test/key2.txt | 0 .../data/testdata/blob_test/key3.txt | 0 .../data/testdata/blob_test/subdir/key4.txt | 0 cron/{internal => }/data/testdata/comment.csv | 0 .../data/testdata/empty_row.csv | 0 .../data/testdata/extra_column.csv | 0 .../data/testdata/failing_urls.csv | 0 .../data/testdata/ignore_header.csv | 0 .../data/testdata/no_header.csv | 0 .../data/testdata/only_header.csv | 0 .../data/testdata/split_file.csv | 0 .../data/testdata/split_file_empty.csv | 0 cron/{internal => }/data/writer.go | 0 cron/{internal => }/data/writer_test.go | 0 cron/internal/bq/main.go | 4 +- cron/internal/cii/main.go | 4 +- cron/internal/controller/main.go | 4 +- cron/internal/data/add/main.go | 2 +- cron/internal/data/add/main_test.go | 2 +- cron/internal/data/metadata.pb.go | 186 --------------- cron/internal/data/update/dependency.go | 2 +- cron/internal/data/update/main.go | 2 +- cron/internal/data/validate/main.go | 2 +- cron/internal/monitoring/exporter.go | 2 +- cron/internal/pubsub/publisher.go | 2 +- cron/internal/pubsub/publisher_test.go | 2 +- cron/internal/pubsub/subscriber.go | 2 +- cron/internal/pubsub/subscriber_gcs.go | 2 +- cron/internal/pubsub/subscriber_gocloud.go | 2 +- .../pubsub/subscriber_gocloud_test.go | 2 +- cron/internal/shuffle/main.go | 2 +- cron/internal/webhook/main.go | 2 +- cron/internal/worker/main.go | 223 +++++++----------- cron/k8s/README.md | 2 +- cron/worker/worker.go | 160 +++++++++++++ cron/worker/worker_test.go | 57 +++++ 56 files changed, 586 insertions(+), 419 deletions(-) rename cron/{internal => }/config/config.go (100%) rename cron/{internal => }/config/config.yaml (100%) rename cron/{internal => }/config/config_test.go (100%) rename cron/{internal => }/config/testdata/basic.yaml (100%) rename cron/{internal => }/config/testdata/missing_field.yaml (100%) rename cron/{internal => }/config/testdata/optional_maps.yaml (100%) rename cron/{internal => }/data/README.md (100%) rename cron/{internal => }/data/blob.go (99%) rename cron/{internal => }/data/blob_test.go (100%) rename cron/{internal => }/data/format.go (100%) rename cron/{internal => }/data/format_test.go (100%) rename cron/{internal => }/data/iterator.go (100%) rename cron/{internal => }/data/iterator_test.go (100%) create mode 100644 cron/data/metadata.pb.go rename cron/{internal => }/data/metadata.proto (92%) rename cron/{internal => }/data/request.pb.go (52%) rename cron/{internal => }/data/request.proto (93%) rename cron/{internal => }/data/testdata/basic.csv (100%) rename cron/{internal => }/data/testdata/blob_test/key1.txt (100%) rename cron/{internal => }/data/testdata/blob_test/key2.txt (100%) rename cron/{internal => }/data/testdata/blob_test/key3.txt (100%) rename cron/{internal => }/data/testdata/blob_test/subdir/key4.txt (100%) rename cron/{internal => }/data/testdata/comment.csv (100%) rename cron/{internal => }/data/testdata/empty_row.csv (100%) rename cron/{internal => }/data/testdata/extra_column.csv (100%) rename cron/{internal => }/data/testdata/failing_urls.csv (100%) rename cron/{internal => }/data/testdata/ignore_header.csv (100%) rename cron/{internal => }/data/testdata/no_header.csv (100%) rename cron/{internal => }/data/testdata/only_header.csv (100%) rename cron/{internal => }/data/testdata/split_file.csv (100%) rename cron/{internal => }/data/testdata/split_file_empty.csv (100%) rename cron/{internal => }/data/writer.go (100%) rename cron/{internal => }/data/writer_test.go (100%) delete mode 100644 cron/internal/data/metadata.pb.go create mode 100644 cron/worker/worker.go create mode 100644 cron/worker/worker_test.go diff --git a/Makefile b/Makefile index 9682d266c37..a36aa11b8ce 100644 --- a/Makefile +++ b/Makefile @@ -129,11 +129,11 @@ build: ## Build all binaries and images in the repo. build: $(build-targets) build-proto: ## Compiles and generates all required protobufs -build-proto: cron/internal/data/request.pb.go cron/internal/data/metadata.pb.go -cron/internal/data/request.pb.go: cron/internal/data/request.proto | $(PROTOC) $(PROTOC_GEN_GO) - $(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=. --go_opt=paths=source_relative cron/internal/data/request.proto -cron/internal/data/metadata.pb.go: cron/internal/data/metadata.proto | $(PROTOC) $(PROTOC_GEN_GO) - $(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=. --go_opt=paths=source_relative cron/internal/data/metadata.proto +build-proto: cron/data/request.pb.go cron/data/metadata.pb.go +cron/data/request.pb.go: cron/data/request.proto | $(PROTOC) $(PROTOC_GEN_GO) + $(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=. --go_opt=paths=source_relative cron/data/request.proto +cron/data/metadata.pb.go: cron/data/metadata.proto | $(PROTOC) $(PROTOC_GEN_GO) + $(PROTOC) --plugin=$(PROTOC_GEN_GO) --go_out=. --go_opt=paths=source_relative cron/data/metadata.proto generate-mocks: ## Compiles and generates all mocks using mockgen. generate-mocks: clients/mockclients/repo_client.go \ @@ -219,14 +219,14 @@ cron/internal/cii/cii-worker.docker: cron/internal/cii/Dockerfile $(CRON_CII_DEP --tag $(IMAGE_NAME)-cii-worker && \ touch cron/internal/cii/cii-worker.docker -CRON_SHUFFLER_DEPS = $(shell find cron/internal/data/ cron/internal/shuffle/ -iname "*.go") +CRON_SHUFFLER_DEPS = $(shell find cron/data/ cron/internal/shuffle/ -iname "*.go") build-shuffler: ## Build cron shuffle script build-shuffler: cron/internal/shuffle/shuffle cron/internal/shuffle/shuffle: $(CRON_SHUFFLER_DEPS) # Run go build on the cron shuffle script cd cron/internal/shuffle && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o shuffle -CRON_TRANSFER_DEPS = $(shell find cron/internal/data/ cron/internal/config/ cron/internal/bq/ -iname "*.go") +CRON_TRANSFER_DEPS = $(shell find cron/data/ cron/config/ cron/internal/bq/ -iname "*.go") build-bq-transfer: ## Build cron BQ transfer worker build-bq-transfer: cron/internal/bq/data-transfer cron/internal/bq/data-transfer: $(CRON_TRANSFER_DEPS) @@ -255,7 +255,7 @@ clients/githubrepo/roundtripper/tokens/server/github-auth-server.docker: \ --tag ${IMAGE_NAME}-github-server && \ touch clients/githubrepo/roundtripper/tokens/server/github-auth-server.docker -CRON_WEBHOOK_DEPS = $(shell find cron/internal/webhook/ cron/internal/data/ -iname "*.go") +CRON_WEBHOOK_DEPS = $(shell find cron/internal/webhook/ cron/data/ -iname "*.go") build-webhook: ## Build cron webhook server build-webhook: cron/internal/webhook/webhook cron/internal/webhook/webhook: $(CRON_WEBHOOK_DEPS) @@ -271,19 +271,19 @@ cron/internal/webhook/webhook.docker: cron/internal/webhook/Dockerfile $(CRON_WE build-add-script: ## Runs go build on the add script build-add-script: cron/internal/data/add/add -cron/internal/data/add/add: cron/internal/data/add/*.go cron/internal/data/*.go cron/internal/data/projects.csv +cron/internal/data/add/add: cron/internal/data/add/*.go cron/data/*.go cron/internal/data/projects.csv # Run go build on the add script cd cron/internal/data/add && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o add build-validate-script: ## Runs go build on the validate script build-validate-script: cron/internal/data/validate/validate -cron/internal/data/validate/validate: cron/internal/data/validate/*.go cron/internal/data/*.go cron/internal/data/projects.csv +cron/internal/data/validate/validate: cron/internal/data/validate/*.go cron/data/*.go cron/internal/data/projects.csv # Run go build on the validate script cd cron/internal/data/validate && CGO_ENABLED=0 go build -trimpath -a -ldflags '$(LDFLAGS)' -o validate build-update-script: ## Runs go build on the update script build-update-script: cron/internal/data/update/projects-update -cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/internal/data/*.go +cron/internal/data/update/projects-update: cron/internal/data/update/*.go cron/data/*.go # Run go build on the update script cd cron/internal/data/update && CGO_ENABLED=0 go build -trimpath -a -tags netgo -ldflags '$(LDFLAGS)' -o projects-update diff --git a/cron/internal/config/config.go b/cron/config/config.go similarity index 100% rename from cron/internal/config/config.go rename to cron/config/config.go diff --git a/cron/internal/config/config.yaml b/cron/config/config.yaml similarity index 100% rename from cron/internal/config/config.yaml rename to cron/config/config.yaml diff --git a/cron/internal/config/config_test.go b/cron/config/config_test.go similarity index 100% rename from cron/internal/config/config_test.go rename to cron/config/config_test.go diff --git a/cron/internal/config/testdata/basic.yaml b/cron/config/testdata/basic.yaml similarity index 100% rename from cron/internal/config/testdata/basic.yaml rename to cron/config/testdata/basic.yaml diff --git a/cron/internal/config/testdata/missing_field.yaml b/cron/config/testdata/missing_field.yaml similarity index 100% rename from cron/internal/config/testdata/missing_field.yaml rename to cron/config/testdata/missing_field.yaml diff --git a/cron/internal/config/testdata/optional_maps.yaml b/cron/config/testdata/optional_maps.yaml similarity index 100% rename from cron/internal/config/testdata/optional_maps.yaml rename to cron/config/testdata/optional_maps.yaml diff --git a/cron/internal/data/README.md b/cron/data/README.md similarity index 100% rename from cron/internal/data/README.md rename to cron/data/README.md diff --git a/cron/internal/data/blob.go b/cron/data/blob.go similarity index 99% rename from cron/internal/data/blob.go rename to cron/data/blob.go index b4d082ec561..94097f57621 100644 --- a/cron/internal/data/blob.go +++ b/cron/data/blob.go @@ -28,7 +28,7 @@ import ( // Needed to link in GCP drivers. _ "gocloud.dev/blob/gcsblob" - "github.com/ossf/scorecard/v4/cron/internal/config" + "github.com/ossf/scorecard/v4/cron/config" ) const ( diff --git a/cron/internal/data/blob_test.go b/cron/data/blob_test.go similarity index 100% rename from cron/internal/data/blob_test.go rename to cron/data/blob_test.go diff --git a/cron/internal/data/format.go b/cron/data/format.go similarity index 100% rename from cron/internal/data/format.go rename to cron/data/format.go diff --git a/cron/internal/data/format_test.go b/cron/data/format_test.go similarity index 100% rename from cron/internal/data/format_test.go rename to cron/data/format_test.go diff --git a/cron/internal/data/iterator.go b/cron/data/iterator.go similarity index 100% rename from cron/internal/data/iterator.go rename to cron/data/iterator.go diff --git a/cron/internal/data/iterator_test.go b/cron/data/iterator_test.go similarity index 100% rename from cron/internal/data/iterator_test.go rename to cron/data/iterator_test.go diff --git a/cron/data/metadata.pb.go b/cron/data/metadata.pb.go new file mode 100644 index 00000000000..938deaae53f --- /dev/null +++ b/cron/data/metadata.pb.go @@ -0,0 +1,185 @@ +// Copyright 2021 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.6 +// source: cron/data/metadata.proto + +package data + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ShardMetadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ShardLoc *string `protobuf:"bytes,1,opt,name=shard_loc,json=shardLoc,proto3,oneof" json:"shard_loc,omitempty"` + NumShard *int32 `protobuf:"varint,2,opt,name=num_shard,json=numShard,proto3,oneof" json:"num_shard,omitempty"` + CommitSha *string `protobuf:"bytes,3,opt,name=commit_sha,json=commitSha,proto3,oneof" json:"commit_sha,omitempty"` +} + +func (x *ShardMetadata) Reset() { + *x = ShardMetadata{} + if protoimpl.UnsafeEnabled { + mi := &file_cron_data_metadata_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ShardMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShardMetadata) ProtoMessage() {} + +func (x *ShardMetadata) ProtoReflect() protoreflect.Message { + mi := &file_cron_data_metadata_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShardMetadata.ProtoReflect.Descriptor instead. +func (*ShardMetadata) Descriptor() ([]byte, []int) { + return file_cron_data_metadata_proto_rawDescGZIP(), []int{0} +} + +func (x *ShardMetadata) GetShardLoc() string { + if x != nil && x.ShardLoc != nil { + return *x.ShardLoc + } + return "" +} + +func (x *ShardMetadata) GetNumShard() int32 { + if x != nil && x.NumShard != nil { + return *x.NumShard + } + return 0 +} + +func (x *ShardMetadata) GetCommitSha() string { + if x != nil && x.CommitSha != nil { + return *x.CommitSha + } + return "" +} + +var File_cron_data_metadata_proto protoreflect.FileDescriptor + +var file_cron_data_metadata_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x6d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x6f, 0x73, 0x73, 0x66, + 0x2e, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x22, 0xa2, 0x01, + 0x0a, 0x0d, 0x53, 0x68, 0x61, 0x72, 0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, + 0x20, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4c, 0x6f, 0x63, 0x88, 0x01, + 0x01, 0x12, 0x20, 0x0a, 0x09, 0x6e, 0x75, 0x6d, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x08, 0x6e, 0x75, 0x6d, 0x53, 0x68, 0x61, 0x72, 0x64, + 0x88, 0x01, 0x01, 0x12, 0x22, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x68, + 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x69, + 0x74, 0x53, 0x68, 0x61, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x68, 0x61, 0x72, + 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x73, + 0x68, 0x61, 0x42, 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6f, 0x73, 0x73, 0x66, 0x2f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2f, + 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_cron_data_metadata_proto_rawDescOnce sync.Once + file_cron_data_metadata_proto_rawDescData = file_cron_data_metadata_proto_rawDesc +) + +func file_cron_data_metadata_proto_rawDescGZIP() []byte { + file_cron_data_metadata_proto_rawDescOnce.Do(func() { + file_cron_data_metadata_proto_rawDescData = protoimpl.X.CompressGZIP(file_cron_data_metadata_proto_rawDescData) + }) + return file_cron_data_metadata_proto_rawDescData +} + +var file_cron_data_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_cron_data_metadata_proto_goTypes = []interface{}{ + (*ShardMetadata)(nil), // 0: ossf.scorecard.cron.internal.data.ShardMetadata +} +var file_cron_data_metadata_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_cron_data_metadata_proto_init() } +func file_cron_data_metadata_proto_init() { + if File_cron_data_metadata_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_cron_data_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ShardMetadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_cron_data_metadata_proto_msgTypes[0].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cron_data_metadata_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_cron_data_metadata_proto_goTypes, + DependencyIndexes: file_cron_data_metadata_proto_depIdxs, + MessageInfos: file_cron_data_metadata_proto_msgTypes, + }.Build() + File_cron_data_metadata_proto = out.File + file_cron_data_metadata_proto_rawDesc = nil + file_cron_data_metadata_proto_goTypes = nil + file_cron_data_metadata_proto_depIdxs = nil +} diff --git a/cron/internal/data/metadata.proto b/cron/data/metadata.proto similarity index 92% rename from cron/internal/data/metadata.proto rename to cron/data/metadata.proto index a215278dc48..38caf1e1c52 100644 --- a/cron/internal/data/metadata.proto +++ b/cron/data/metadata.proto @@ -16,7 +16,7 @@ syntax = "proto3"; package ossf.scorecard.cron.internal.data; -option go_package = "github.com/ossf/scorecard/cron/internal/data"; +option go_package = "github.com/ossf/scorecard/cron/data"; message ShardMetadata { optional string shard_loc = 1; diff --git a/cron/internal/data/request.pb.go b/cron/data/request.pb.go similarity index 52% rename from cron/internal/data/request.pb.go rename to cron/data/request.pb.go index 9d141fc2fa5..175572a82b8 100644 --- a/cron/internal/data/request.pb.go +++ b/cron/data/request.pb.go @@ -16,7 +16,7 @@ // versions: // protoc-gen-go v1.28.1 // protoc v3.21.6 -// source: cron/internal/data/request.proto +// source: cron/data/request.proto package data @@ -48,7 +48,7 @@ type Repo struct { func (x *Repo) Reset() { *x = Repo{} if protoimpl.UnsafeEnabled { - mi := &file_cron_internal_data_request_proto_msgTypes[0] + mi := &file_cron_data_request_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -61,7 +61,7 @@ func (x *Repo) String() string { func (*Repo) ProtoMessage() {} func (x *Repo) ProtoReflect() protoreflect.Message { - mi := &file_cron_internal_data_request_proto_msgTypes[0] + mi := &file_cron_data_request_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -74,7 +74,7 @@ func (x *Repo) ProtoReflect() protoreflect.Message { // Deprecated: Use Repo.ProtoReflect.Descriptor instead. func (*Repo) Descriptor() ([]byte, []int) { - return file_cron_internal_data_request_proto_rawDescGZIP(), []int{0} + return file_cron_data_request_proto_rawDescGZIP(), []int{0} } func (x *Repo) GetUrl() string { @@ -111,7 +111,7 @@ type ScorecardBatchRequest struct { func (x *ScorecardBatchRequest) Reset() { *x = ScorecardBatchRequest{} if protoimpl.UnsafeEnabled { - mi := &file_cron_internal_data_request_proto_msgTypes[1] + mi := &file_cron_data_request_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -124,7 +124,7 @@ func (x *ScorecardBatchRequest) String() string { func (*ScorecardBatchRequest) ProtoMessage() {} func (x *ScorecardBatchRequest) ProtoReflect() protoreflect.Message { - mi := &file_cron_internal_data_request_proto_msgTypes[1] + mi := &file_cron_data_request_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -137,7 +137,7 @@ func (x *ScorecardBatchRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ScorecardBatchRequest.ProtoReflect.Descriptor instead. func (*ScorecardBatchRequest) Descriptor() ([]byte, []int) { - return file_cron_internal_data_request_proto_rawDescGZIP(), []int{1} + return file_cron_data_request_proto_rawDescGZIP(), []int{1} } func (x *ScorecardBatchRequest) GetRepos() []*Repo { @@ -161,61 +161,59 @@ func (x *ScorecardBatchRequest) GetJobTime() *timestamppb.Timestamp { return nil } -var File_cron_internal_data_request_proto protoreflect.FileDescriptor +var File_cron_data_request_proto protoreflect.FileDescriptor -var file_cron_internal_data_request_proto_rawDesc = []byte{ - 0x0a, 0x20, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, - 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x12, 0x21, 0x6f, 0x73, 0x73, 0x66, 0x2e, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, +var file_cron_data_request_proto_rawDesc = []byte{ + 0x0a, 0x17, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x6f, 0x73, 0x73, 0x66, 0x2e, + 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x1f, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x69, 0x0a, + 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x15, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, + 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x06, + 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x88, 0x01, 0x01, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x09, 0x0a, + 0x07, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x22, 0xd5, 0x01, 0x0a, 0x15, 0x53, 0x63, 0x6f, + 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x73, 0x73, 0x66, 0x2e, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x2e, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x69, 0x0a, 0x04, 0x52, 0x65, 0x70, 0x6f, 0x12, 0x15, - 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x03, 0x75, - 0x72, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x88, - 0x01, 0x01, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x06, - 0x0a, 0x04, 0x5f, 0x75, 0x72, 0x6c, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, - 0x74, 0x22, 0xd5, 0x01, 0x0a, 0x15, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x05, 0x72, - 0x65, 0x70, 0x6f, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6f, 0x73, 0x73, - 0x66, 0x2e, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, - 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, - 0x65, 0x70, 0x6f, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x68, - 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, - 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x08, - 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x01, 0x52, 0x07, 0x6a, 0x6f, - 0x62, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x68, 0x61, - 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x73, 0x73, 0x66, 0x2f, 0x73, 0x63, 0x6f, - 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2f, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x2e, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x52, 0x05, 0x72, 0x65, 0x70, 0x6f, + 0x73, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x73, 0x68, 0x61, 0x72, 0x64, 0x4e, 0x75, 0x6d, + 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x08, 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x48, 0x01, 0x52, 0x07, 0x6a, 0x6f, 0x62, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x42, + 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6e, 0x75, 0x6d, 0x42, 0x0b, 0x0a, + 0x09, 0x5f, 0x6a, 0x6f, 0x62, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, + 0x42, 0x25, 0x5a, 0x23, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, + 0x73, 0x73, 0x66, 0x2f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2f, 0x63, 0x72, + 0x6f, 0x6e, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( - file_cron_internal_data_request_proto_rawDescOnce sync.Once - file_cron_internal_data_request_proto_rawDescData = file_cron_internal_data_request_proto_rawDesc + file_cron_data_request_proto_rawDescOnce sync.Once + file_cron_data_request_proto_rawDescData = file_cron_data_request_proto_rawDesc ) -func file_cron_internal_data_request_proto_rawDescGZIP() []byte { - file_cron_internal_data_request_proto_rawDescOnce.Do(func() { - file_cron_internal_data_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_cron_internal_data_request_proto_rawDescData) +func file_cron_data_request_proto_rawDescGZIP() []byte { + file_cron_data_request_proto_rawDescOnce.Do(func() { + file_cron_data_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_cron_data_request_proto_rawDescData) }) - return file_cron_internal_data_request_proto_rawDescData + return file_cron_data_request_proto_rawDescData } -var file_cron_internal_data_request_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_cron_internal_data_request_proto_goTypes = []interface{}{ +var file_cron_data_request_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_cron_data_request_proto_goTypes = []interface{}{ (*Repo)(nil), // 0: ossf.scorecard.cron.internal.data.Repo (*ScorecardBatchRequest)(nil), // 1: ossf.scorecard.cron.internal.data.ScorecardBatchRequest (*timestamppb.Timestamp)(nil), // 2: google.protobuf.Timestamp } -var file_cron_internal_data_request_proto_depIdxs = []int32{ +var file_cron_data_request_proto_depIdxs = []int32{ 0, // 0: ossf.scorecard.cron.internal.data.ScorecardBatchRequest.repos:type_name -> ossf.scorecard.cron.internal.data.Repo 2, // 1: ossf.scorecard.cron.internal.data.ScorecardBatchRequest.job_time:type_name -> google.protobuf.Timestamp 2, // [2:2] is the sub-list for method output_type @@ -225,13 +223,13 @@ var file_cron_internal_data_request_proto_depIdxs = []int32{ 0, // [0:2] is the sub-list for field type_name } -func init() { file_cron_internal_data_request_proto_init() } -func file_cron_internal_data_request_proto_init() { - if File_cron_internal_data_request_proto != nil { +func init() { file_cron_data_request_proto_init() } +func file_cron_data_request_proto_init() { + if File_cron_data_request_proto != nil { return } if !protoimpl.UnsafeEnabled { - file_cron_internal_data_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_cron_data_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Repo); i { case 0: return &v.state @@ -243,7 +241,7 @@ func file_cron_internal_data_request_proto_init() { return nil } } - file_cron_internal_data_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_cron_data_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ScorecardBatchRequest); i { case 0: return &v.state @@ -256,24 +254,24 @@ func file_cron_internal_data_request_proto_init() { } } } - file_cron_internal_data_request_proto_msgTypes[0].OneofWrappers = []interface{}{} - file_cron_internal_data_request_proto_msgTypes[1].OneofWrappers = []interface{}{} + file_cron_data_request_proto_msgTypes[0].OneofWrappers = []interface{}{} + file_cron_data_request_proto_msgTypes[1].OneofWrappers = []interface{}{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_cron_internal_data_request_proto_rawDesc, + RawDescriptor: file_cron_data_request_proto_rawDesc, NumEnums: 0, NumMessages: 2, NumExtensions: 0, NumServices: 0, }, - GoTypes: file_cron_internal_data_request_proto_goTypes, - DependencyIndexes: file_cron_internal_data_request_proto_depIdxs, - MessageInfos: file_cron_internal_data_request_proto_msgTypes, + GoTypes: file_cron_data_request_proto_goTypes, + DependencyIndexes: file_cron_data_request_proto_depIdxs, + MessageInfos: file_cron_data_request_proto_msgTypes, }.Build() - File_cron_internal_data_request_proto = out.File - file_cron_internal_data_request_proto_rawDesc = nil - file_cron_internal_data_request_proto_goTypes = nil - file_cron_internal_data_request_proto_depIdxs = nil + File_cron_data_request_proto = out.File + file_cron_data_request_proto_rawDesc = nil + file_cron_data_request_proto_goTypes = nil + file_cron_data_request_proto_depIdxs = nil } diff --git a/cron/internal/data/request.proto b/cron/data/request.proto similarity index 93% rename from cron/internal/data/request.proto rename to cron/data/request.proto index 8c6a7c7598a..ac3718ec879 100644 --- a/cron/internal/data/request.proto +++ b/cron/data/request.proto @@ -18,7 +18,7 @@ package ossf.scorecard.cron.internal.data; import "google/protobuf/timestamp.proto"; -option go_package = "github.com/ossf/scorecard/cron/internal/data"; +option go_package = "github.com/ossf/scorecard/cron/data"; message Repo { optional string url = 1; diff --git a/cron/internal/data/testdata/basic.csv b/cron/data/testdata/basic.csv similarity index 100% rename from cron/internal/data/testdata/basic.csv rename to cron/data/testdata/basic.csv diff --git a/cron/internal/data/testdata/blob_test/key1.txt b/cron/data/testdata/blob_test/key1.txt similarity index 100% rename from cron/internal/data/testdata/blob_test/key1.txt rename to cron/data/testdata/blob_test/key1.txt diff --git a/cron/internal/data/testdata/blob_test/key2.txt b/cron/data/testdata/blob_test/key2.txt similarity index 100% rename from cron/internal/data/testdata/blob_test/key2.txt rename to cron/data/testdata/blob_test/key2.txt diff --git a/cron/internal/data/testdata/blob_test/key3.txt b/cron/data/testdata/blob_test/key3.txt similarity index 100% rename from cron/internal/data/testdata/blob_test/key3.txt rename to cron/data/testdata/blob_test/key3.txt diff --git a/cron/internal/data/testdata/blob_test/subdir/key4.txt b/cron/data/testdata/blob_test/subdir/key4.txt similarity index 100% rename from cron/internal/data/testdata/blob_test/subdir/key4.txt rename to cron/data/testdata/blob_test/subdir/key4.txt diff --git a/cron/internal/data/testdata/comment.csv b/cron/data/testdata/comment.csv similarity index 100% rename from cron/internal/data/testdata/comment.csv rename to cron/data/testdata/comment.csv diff --git a/cron/internal/data/testdata/empty_row.csv b/cron/data/testdata/empty_row.csv similarity index 100% rename from cron/internal/data/testdata/empty_row.csv rename to cron/data/testdata/empty_row.csv diff --git a/cron/internal/data/testdata/extra_column.csv b/cron/data/testdata/extra_column.csv similarity index 100% rename from cron/internal/data/testdata/extra_column.csv rename to cron/data/testdata/extra_column.csv diff --git a/cron/internal/data/testdata/failing_urls.csv b/cron/data/testdata/failing_urls.csv similarity index 100% rename from cron/internal/data/testdata/failing_urls.csv rename to cron/data/testdata/failing_urls.csv diff --git a/cron/internal/data/testdata/ignore_header.csv b/cron/data/testdata/ignore_header.csv similarity index 100% rename from cron/internal/data/testdata/ignore_header.csv rename to cron/data/testdata/ignore_header.csv diff --git a/cron/internal/data/testdata/no_header.csv b/cron/data/testdata/no_header.csv similarity index 100% rename from cron/internal/data/testdata/no_header.csv rename to cron/data/testdata/no_header.csv diff --git a/cron/internal/data/testdata/only_header.csv b/cron/data/testdata/only_header.csv similarity index 100% rename from cron/internal/data/testdata/only_header.csv rename to cron/data/testdata/only_header.csv diff --git a/cron/internal/data/testdata/split_file.csv b/cron/data/testdata/split_file.csv similarity index 100% rename from cron/internal/data/testdata/split_file.csv rename to cron/data/testdata/split_file.csv diff --git a/cron/internal/data/testdata/split_file_empty.csv b/cron/data/testdata/split_file_empty.csv similarity index 100% rename from cron/internal/data/testdata/split_file_empty.csv rename to cron/data/testdata/split_file_empty.csv diff --git a/cron/internal/data/writer.go b/cron/data/writer.go similarity index 100% rename from cron/internal/data/writer.go rename to cron/data/writer.go diff --git a/cron/internal/data/writer_test.go b/cron/data/writer_test.go similarity index 100% rename from cron/internal/data/writer_test.go rename to cron/data/writer_test.go diff --git a/cron/internal/bq/main.go b/cron/internal/bq/main.go index c30c9c86dd5..8a6401fba4d 100644 --- a/cron/internal/bq/main.go +++ b/cron/internal/bq/main.go @@ -28,8 +28,8 @@ import ( "google.golang.org/protobuf/encoding/protojson" - "github.com/ossf/scorecard/v4/cron/internal/config" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/config" + "github.com/ossf/scorecard/v4/cron/data" ) type shardSummary struct { diff --git a/cron/internal/cii/main.go b/cron/internal/cii/main.go index 983a69bc189..cece18af447 100644 --- a/cron/internal/cii/main.go +++ b/cron/internal/cii/main.go @@ -25,8 +25,8 @@ import ( "strings" "github.com/ossf/scorecard/v4/clients" - "github.com/ossf/scorecard/v4/cron/internal/config" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/config" + "github.com/ossf/scorecard/v4/cron/data" ) const ciiBaseURL = "https://bestpractices.coreinfrastructure.org/projects.json" diff --git a/cron/internal/controller/main.go b/cron/internal/controller/main.go index aaef1f67920..da37de62c8e 100644 --- a/cron/internal/controller/main.go +++ b/cron/internal/controller/main.go @@ -28,8 +28,8 @@ import ( "sigs.k8s.io/release-utils/version" "github.com/ossf/scorecard/v4/clients" - "github.com/ossf/scorecard/v4/cron/internal/config" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/config" + "github.com/ossf/scorecard/v4/cron/data" "github.com/ossf/scorecard/v4/cron/internal/pubsub" ) diff --git a/cron/internal/data/add/main.go b/cron/internal/data/add/main.go index ce435cba96f..d214ef1b9fe 100644 --- a/cron/internal/data/add/main.go +++ b/cron/internal/data/add/main.go @@ -22,7 +22,7 @@ import ( "os" "strings" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) // Script to add new project repositories to the projects.csv file: diff --git a/cron/internal/data/add/main_test.go b/cron/internal/data/add/main_test.go index 08e51792d50..27a9c04291a 100644 --- a/cron/internal/data/add/main_test.go +++ b/cron/internal/data/add/main_test.go @@ -22,7 +22,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) func lessThanURI(x, y data.RepoFormat) bool { diff --git a/cron/internal/data/metadata.pb.go b/cron/internal/data/metadata.pb.go deleted file mode 100644 index 1e260e46354..00000000000 --- a/cron/internal/data/metadata.pb.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2021 Security Scorecard Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.1 -// protoc v3.21.6 -// source: cron/internal/data/metadata.proto - -package data - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type ShardMetadata struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ShardLoc *string `protobuf:"bytes,1,opt,name=shard_loc,json=shardLoc,proto3,oneof" json:"shard_loc,omitempty"` - NumShard *int32 `protobuf:"varint,2,opt,name=num_shard,json=numShard,proto3,oneof" json:"num_shard,omitempty"` - CommitSha *string `protobuf:"bytes,3,opt,name=commit_sha,json=commitSha,proto3,oneof" json:"commit_sha,omitempty"` -} - -func (x *ShardMetadata) Reset() { - *x = ShardMetadata{} - if protoimpl.UnsafeEnabled { - mi := &file_cron_internal_data_metadata_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ShardMetadata) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ShardMetadata) ProtoMessage() {} - -func (x *ShardMetadata) ProtoReflect() protoreflect.Message { - mi := &file_cron_internal_data_metadata_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ShardMetadata.ProtoReflect.Descriptor instead. -func (*ShardMetadata) Descriptor() ([]byte, []int) { - return file_cron_internal_data_metadata_proto_rawDescGZIP(), []int{0} -} - -func (x *ShardMetadata) GetShardLoc() string { - if x != nil && x.ShardLoc != nil { - return *x.ShardLoc - } - return "" -} - -func (x *ShardMetadata) GetNumShard() int32 { - if x != nil && x.NumShard != nil { - return *x.NumShard - } - return 0 -} - -func (x *ShardMetadata) GetCommitSha() string { - if x != nil && x.CommitSha != nil { - return *x.CommitSha - } - return "" -} - -var File_cron_internal_data_metadata_proto protoreflect.FileDescriptor - -var file_cron_internal_data_metadata_proto_rawDesc = []byte{ - 0x0a, 0x21, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, - 0x64, 0x61, 0x74, 0x61, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x21, 0x6f, 0x73, 0x73, 0x66, 0x2e, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x63, - 0x61, 0x72, 0x64, 0x2e, 0x63, 0x72, 0x6f, 0x6e, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x6c, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x22, 0xa2, 0x01, 0x0a, 0x0d, 0x53, 0x68, 0x61, 0x72, 0x64, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x20, 0x0a, 0x09, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x08, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x4c, 0x6f, 0x63, 0x88, 0x01, 0x01, 0x12, 0x20, 0x0a, 0x09, 0x6e, 0x75, - 0x6d, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, - 0x08, 0x6e, 0x75, 0x6d, 0x53, 0x68, 0x61, 0x72, 0x64, 0x88, 0x01, 0x01, 0x12, 0x22, 0x0a, 0x0a, - 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x68, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, - 0x48, 0x02, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x53, 0x68, 0x61, 0x88, 0x01, 0x01, - 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x6c, 0x6f, 0x63, 0x42, 0x0c, - 0x0a, 0x0a, 0x5f, 0x6e, 0x75, 0x6d, 0x5f, 0x73, 0x68, 0x61, 0x72, 0x64, 0x42, 0x0d, 0x0a, 0x0b, - 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x5f, 0x73, 0x68, 0x61, 0x42, 0x2e, 0x5a, 0x2c, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6f, 0x73, 0x73, 0x66, 0x2f, 0x73, - 0x63, 0x6f, 0x72, 0x65, 0x63, 0x61, 0x72, 0x64, 0x2f, 0x63, 0x72, 0x6f, 0x6e, 0x2f, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, -} - -var ( - file_cron_internal_data_metadata_proto_rawDescOnce sync.Once - file_cron_internal_data_metadata_proto_rawDescData = file_cron_internal_data_metadata_proto_rawDesc -) - -func file_cron_internal_data_metadata_proto_rawDescGZIP() []byte { - file_cron_internal_data_metadata_proto_rawDescOnce.Do(func() { - file_cron_internal_data_metadata_proto_rawDescData = protoimpl.X.CompressGZIP(file_cron_internal_data_metadata_proto_rawDescData) - }) - return file_cron_internal_data_metadata_proto_rawDescData -} - -var file_cron_internal_data_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_cron_internal_data_metadata_proto_goTypes = []interface{}{ - (*ShardMetadata)(nil), // 0: ossf.scorecard.cron.internal.data.ShardMetadata -} -var file_cron_internal_data_metadata_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_cron_internal_data_metadata_proto_init() } -func file_cron_internal_data_metadata_proto_init() { - if File_cron_internal_data_metadata_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_cron_internal_data_metadata_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ShardMetadata); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_cron_internal_data_metadata_proto_msgTypes[0].OneofWrappers = []interface{}{} - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_cron_internal_data_metadata_proto_rawDesc, - NumEnums: 0, - NumMessages: 1, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_cron_internal_data_metadata_proto_goTypes, - DependencyIndexes: file_cron_internal_data_metadata_proto_depIdxs, - MessageInfos: file_cron_internal_data_metadata_proto_msgTypes, - }.Build() - File_cron_internal_data_metadata_proto = out.File - file_cron_internal_data_metadata_proto_rawDesc = nil - file_cron_internal_data_metadata_proto_goTypes = nil - file_cron_internal_data_metadata_proto_depIdxs = nil -} diff --git a/cron/internal/data/update/dependency.go b/cron/internal/data/update/dependency.go index 6081313886e..8f1e13b3cfc 100644 --- a/cron/internal/data/update/dependency.go +++ b/cron/internal/data/update/dependency.go @@ -30,7 +30,7 @@ import ( "golang.org/x/tools/go/vcs" "github.com/ossf/scorecard/v4/clients/githubrepo" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) var ( diff --git a/cron/internal/data/update/main.go b/cron/internal/data/update/main.go index 07d0d914b74..d3d9f49e492 100644 --- a/cron/internal/data/update/main.go +++ b/cron/internal/data/update/main.go @@ -19,7 +19,7 @@ import ( "bytes" "os" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) // Adds "project=${PROJECT},dependency=true" to the repositories metadata. diff --git a/cron/internal/data/validate/main.go b/cron/internal/data/validate/main.go index 47ddac04ecc..aeb8141badc 100644 --- a/cron/internal/data/validate/main.go +++ b/cron/internal/data/validate/main.go @@ -19,7 +19,7 @@ import ( "log" "os" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) // Validates data.Iterator used by production PubSub cron job. diff --git a/cron/internal/monitoring/exporter.go b/cron/internal/monitoring/exporter.go index 3bc5e483638..922ce1b4148 100644 --- a/cron/internal/monitoring/exporter.go +++ b/cron/internal/monitoring/exporter.go @@ -24,7 +24,7 @@ import ( "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource/gcp" "go.opencensus.io/stats/view" - "github.com/ossf/scorecard/v4/cron/internal/config" + "github.com/ossf/scorecard/v4/cron/config" ) var errorUndefinedExporter = errors.New("unsupported exporterType") diff --git a/cron/internal/pubsub/publisher.go b/cron/internal/pubsub/publisher.go index fc4200497c1..6310e166ef9 100644 --- a/cron/internal/pubsub/publisher.go +++ b/cron/internal/pubsub/publisher.go @@ -28,7 +28,7 @@ import ( _ "gocloud.dev/pubsub/gcppubsub" "google.golang.org/protobuf/encoding/protojson" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) var errorPublish = errors.New("total errors when publishing") diff --git a/cron/internal/pubsub/publisher_test.go b/cron/internal/pubsub/publisher_test.go index 39270de5e8b..3f09815756a 100644 --- a/cron/internal/pubsub/publisher_test.go +++ b/cron/internal/pubsub/publisher_test.go @@ -21,7 +21,7 @@ import ( "gocloud.dev/pubsub" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) type mockSucceedTopic struct{} diff --git a/cron/internal/pubsub/subscriber.go b/cron/internal/pubsub/subscriber.go index fcca1a51cd0..86ecff60d9a 100644 --- a/cron/internal/pubsub/subscriber.go +++ b/cron/internal/pubsub/subscriber.go @@ -21,7 +21,7 @@ import ( "google.golang.org/protobuf/encoding/protojson" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) // ErrorInParse indicates there was an error while unmarshalling the protocol buffer message. diff --git a/cron/internal/pubsub/subscriber_gcs.go b/cron/internal/pubsub/subscriber_gcs.go index e3553ba60fa..b9373e1b467 100644 --- a/cron/internal/pubsub/subscriber_gcs.go +++ b/cron/internal/pubsub/subscriber_gcs.go @@ -24,7 +24,7 @@ import ( pubsub "cloud.google.com/go/pubsub/apiv1" pubsubpb "google.golang.org/genproto/googleapis/pubsub/v1" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) const ( diff --git a/cron/internal/pubsub/subscriber_gocloud.go b/cron/internal/pubsub/subscriber_gocloud.go index 5dbd880170f..faaf7ca6d89 100644 --- a/cron/internal/pubsub/subscriber_gocloud.go +++ b/cron/internal/pubsub/subscriber_gocloud.go @@ -23,7 +23,7 @@ import ( // Needed to link in GCP drivers. _ "gocloud.dev/pubsub/gcppubsub" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) type receiver interface { diff --git a/cron/internal/pubsub/subscriber_gocloud_test.go b/cron/internal/pubsub/subscriber_gocloud_test.go index c593cdffecf..88ab12888d0 100644 --- a/cron/internal/pubsub/subscriber_gocloud_test.go +++ b/cron/internal/pubsub/subscriber_gocloud_test.go @@ -23,7 +23,7 @@ import ( "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) var repo1 = "repo1" diff --git a/cron/internal/shuffle/main.go b/cron/internal/shuffle/main.go index 671f8f0f4df..805b0c189d7 100644 --- a/cron/internal/shuffle/main.go +++ b/cron/internal/shuffle/main.go @@ -21,7 +21,7 @@ import ( "strconv" "time" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) func main() { diff --git a/cron/internal/webhook/main.go b/cron/internal/webhook/main.go index 7b7f03bc7d4..42649a7f807 100644 --- a/cron/internal/webhook/main.go +++ b/cron/internal/webhook/main.go @@ -26,7 +26,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/google" "google.golang.org/protobuf/encoding/protojson" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/data" ) const stableTag = "stable" diff --git a/cron/internal/worker/main.go b/cron/internal/worker/main.go index fe6782a1019..2ad727a2aed 100644 --- a/cron/internal/worker/main.go +++ b/cron/internal/worker/main.go @@ -30,11 +30,11 @@ import ( "github.com/ossf/scorecard/v4/clients" "github.com/ossf/scorecard/v4/clients/githubrepo" githubstats "github.com/ossf/scorecard/v4/clients/githubrepo/stats" - "github.com/ossf/scorecard/v4/cron/internal/config" - "github.com/ossf/scorecard/v4/cron/internal/data" + "github.com/ossf/scorecard/v4/cron/config" + "github.com/ossf/scorecard/v4/cron/data" format "github.com/ossf/scorecard/v4/cron/internal/format" "github.com/ossf/scorecard/v4/cron/internal/monitoring" - "github.com/ossf/scorecard/v4/cron/internal/pubsub" + "github.com/ossf/scorecard/v4/cron/worker" docs "github.com/ossf/scorecard/v4/docs/checks" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/log" @@ -50,6 +50,81 @@ const ( var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored") +type ScorecardWorker struct { + ctx context.Context + logger *log.Logger + checkDocs docs.Doc + exporter monitoring.Exporter + repoClient clients.RepoClient + ciiClient clients.CIIBestPracticesClient + ossFuzzRepoClient clients.RepoClient + vulnsClient clients.VulnerabilitiesClient + apiBucketURL string + rawBucketURL string + blacklistedChecks []string +} + +func newScorecardWorker() (*ScorecardWorker, error) { + var err error + sw := &ScorecardWorker{} + if sw.checkDocs, err = docs.Read(); err != nil { + return nil, fmt.Errorf("docs.Read: %w", err) + } + + if sw.rawBucketURL, err = config.GetRawResultDataBucketURL(); err != nil { + return nil, fmt.Errorf("docs.GetRawResultDataBucketURL: %w", err) + } + + if sw.blacklistedChecks, err = config.GetBlacklistedChecks(); err != nil { + return nil, fmt.Errorf("config.GetBlacklistedChecks: %w", err) + } + + var ciiDataBucketURL string + if ciiDataBucketURL, err = config.GetCIIDataBucketURL(); err != nil { + return nil, fmt.Errorf("config.GetCIIDataBucketURL: %w", err) + } + + if sw.apiBucketURL, err = config.GetAPIResultsBucketURL(); err != nil { + return nil, fmt.Errorf("config.GetAPIResultsBucketURL: %w", err) + } + + sw.ctx = context.Background() + sw.logger = log.NewLogger(log.InfoLevel) + sw.repoClient = githubrepo.CreateGithubRepoClient(sw.ctx, sw.logger) + sw.ciiClient = clients.BlobCIIBestPracticesClient(ciiDataBucketURL) + if sw.ossFuzzRepoClient, err = githubrepo.CreateOssFuzzRepoClient(sw.ctx, sw.logger); err != nil { + return nil, fmt.Errorf("githubrepo.CreateOssFuzzRepoClient: %w", err) + } + + sw.vulnsClient = clients.DefaultVulnerabilitiesClient() + + if sw.exporter, err = startMetricsExporter(); err != nil { + return nil, fmt.Errorf("startMetricsExporter: %w", err) + } + + // Exposed for monitoring runtime profiles + go func() { + // TODO(log): Previously Fatal. Need to handle the error here. + //nolint:gosec // not internet facing. + sw.logger.Info(fmt.Sprintf("%v", http.ListenAndServe(":8080", nil))) + }() + return sw, nil +} + +func (sw *ScorecardWorker) Close() { + sw.exporter.StopMetricsExporter() + sw.ossFuzzRepoClient.Close() +} + +func (sw *ScorecardWorker) Process(ctx context.Context, req *data.ScorecardBatchRequest, bucketURL string) error { + return processRequest(ctx, req, sw.blacklistedChecks, bucketURL, sw.rawBucketURL, sw.apiBucketURL, + sw.checkDocs, sw.repoClient, sw.ossFuzzRepoClient, sw.ciiClient, sw.vulnsClient, sw.logger) +} + +func (sw *ScorecardWorker) PostProcess() { + sw.exporter.Flush() +} + //nolint:gocognit func processRequest(ctx context.Context, batchRequest *data.ScorecardBatchRequest, @@ -60,25 +135,7 @@ func processRequest(ctx context.Context, vulnsClient clients.VulnerabilitiesClient, logger *log.Logger, ) error { - filename := data.GetBlobFilename( - fmt.Sprintf("shard-%07d", batchRequest.GetShardNum()), - batchRequest.GetJobTime().AsTime()) - // Sanity check - make sure we are not re-processing an already processed request. - existsScore, err := data.BlobExists(ctx, bucketURL, filename) - if err != nil { - return fmt.Errorf("error during BlobExists: %w", err) - } - - existsRaw, err := data.BlobExists(ctx, rawBucketURL, filename) - if err != nil { - return fmt.Errorf("error during BlobExists: %w", err) - } - - if existsScore && existsRaw { - logger.Info(fmt.Sprintf("Already processed shard %s. Nothing to do.", filename)) - // We have already processed this request, nothing to do. - return nil - } + filename := worker.ResultFilename(batchRequest) var buffer2 bytes.Buffer var rawBuffer bytes.Buffer @@ -171,12 +228,14 @@ func processRequest(ctx context.Context, } } - if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer2.Bytes()); err != nil { + // Raw result. + if err := data.WriteToBlobStore(ctx, rawBucketURL, filename, rawBuffer.Bytes()); err != nil { return fmt.Errorf("error during WriteToBlobStore2: %w", err) } - // Raw result. - if err := data.WriteToBlobStore(ctx, rawBucketURL, filename, rawBuffer.Bytes()); err != nil { + // write to the canonical bucket last, as the presence of filename indicates the job was completed. + // see worker package for details. + if err := data.WriteToBlobStore(ctx, bucketURL, filename, buffer2.Bytes()); err != nil { return fmt.Errorf("error during WriteToBlobStore2: %w", err) } @@ -204,121 +263,15 @@ func startMetricsExporter() (monitoring.Exporter, error) { return exporter, nil } -func hasMetadataFile(ctx context.Context, req *data.ScorecardBatchRequest, bucketURL string) (bool, error) { - filename := data.GetBlobFilename(config.ShardMetadataFilename, req.GetJobTime().AsTime()) - exists, err := data.BlobExists(ctx, bucketURL, filename) - if err != nil { - return false, fmt.Errorf("data.BlobExists: %w", err) - } - return exists, nil -} - func main() { - ctx := context.Background() - flag.Parse() - if err := config.ReadConfig(); err != nil { - panic(err) - } - - checkDocs, err := docs.Read() - if err != nil { - panic(err) - } - - subscriptionURL, err := config.GetRequestSubscriptionURL() - if err != nil { - panic(err) - } - subscriber, err := pubsub.CreateSubscriber(ctx, subscriptionURL) - if err != nil { - panic(err) - } - - bucketURL, err := config.GetResultDataBucketURL() + sw, err := newScorecardWorker() if err != nil { panic(err) } - - rawBucketURL, err := config.GetRawResultDataBucketURL() - if err != nil { - panic(err) - } - - blacklistedChecks, err := config.GetBlacklistedChecks() - if err != nil { - panic(err) - } - - ciiDataBucketURL, err := config.GetCIIDataBucketURL() - if err != nil { - panic(err) - } - - apiBucketURL, err := config.GetAPIResultsBucketURL() - if err != nil { - panic(err) - } - - logger := log.NewLogger(log.InfoLevel) - repoClient := githubrepo.CreateGithubRepoClient(ctx, logger) - ciiClient := clients.BlobCIIBestPracticesClient(ciiDataBucketURL) - ossFuzzRepoClient, err := githubrepo.CreateOssFuzzRepoClient(ctx, logger) - vulnsClient := clients.DefaultVulnerabilitiesClient() - if err != nil { - panic(err) - } - defer ossFuzzRepoClient.Close() - - exporter, err := startMetricsExporter() - if err != nil { - panic(err) - } - defer exporter.StopMetricsExporter() - - // Exposed for monitoring runtime profiles - go func() { - // TODO(log): Previously Fatal. Need to handle the error here. - //nolint: gosec // internal server. - logger.Info(fmt.Sprintf("%v", http.ListenAndServe(":8080", nil))) - }() - - for { - req, err := subscriber.SynchronousPull() - if err != nil { - panic(err) - } - - logger.Info("Received message from subscription") - if req == nil { - // TODO(log): Previously Warn. Consider logging an error here. - logger.Info("subscription returned nil message during Receive, exiting") - break - } - - // don't process requests from jobs without metadata files, as the results will never be transferred. - // https://github.com/ossf/scorecard/issues/2307 - if hasMd, err := hasMetadataFile(ctx, req, bucketURL); !hasMd || err != nil { - // nack the message so it can be tried later, as the metadata file may not have been created yet. - subscriber.Nack() - continue - } - - if err := processRequest(ctx, req, blacklistedChecks, - bucketURL, rawBucketURL, apiBucketURL, checkDocs, - repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil { - // TODO(log): Previously Warn. Consider logging an error here. - logger.Info(fmt.Sprintf("error processing request: %v", err)) - // Nack the message so that another worker can retry. - subscriber.Nack() - continue - } - - exporter.Flush() - subscriber.Ack() - } - err = subscriber.Close() - if err != nil { + defer sw.Close() + wl := worker.NewWorkLoop(sw) + if err := wl.Run(); err != nil { panic(err) } } diff --git a/cron/k8s/README.md b/cron/k8s/README.md index 314fd5271a5..aee2f7fe1da 100644 --- a/cron/k8s/README.md +++ b/cron/k8s/README.md @@ -25,7 +25,7 @@ The cluster name is `openssf` which is in zone `us-central1-c`. ## Creating or updating the ConfigMap using the config.yaml file -We use [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/) to store our config file (`cron/internal/config/config.yaml`). +We use [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/) to store our config file (`cron/config/config.yaml`). The file can be created for the first time, or updated, with the same command: ``` kubectl create configmap scorecard-config --from-file=config.yaml -o yaml --dry-run=client | kubectl apply -f - diff --git a/cron/worker/worker.go b/cron/worker/worker.go new file mode 100644 index 00000000000..71b57d4fcbf --- /dev/null +++ b/cron/worker/worker.go @@ -0,0 +1,160 @@ +// Copyright 2022 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package worker implements the generic cron worker logic. +package worker + +import ( + "context" + "flag" + "fmt" + + "github.com/ossf/scorecard/v4/cron/config" + "github.com/ossf/scorecard/v4/cron/data" + "github.com/ossf/scorecard/v4/cron/internal/pubsub" + "github.com/ossf/scorecard/v4/log" +) + +// Worker is the interface used to process batch requests. +// +// Process does the processing for a batch request. Returning an error will cause the request to be nack'd, +// allowing it to be re-processed later. If no error is returned, the request will be ack'd, consuming it. +// +// PostProcess is called only after an error-free call to Process. +type Worker interface { + Process(ctx context.Context, req *data.ScorecardBatchRequest, bucketURL string) error + PostProcess() +} + +// WorkLoop is the entry point into the common cron worker structure. +type WorkLoop struct { + worker Worker +} + +// NewWorkLoop creates a workloop using a specified worker. +func NewWorkLoop(worker Worker) WorkLoop { + return WorkLoop{worker: worker} +} + +// Run initiates the processing performed by the WorkLoop. +func (wl *WorkLoop) Run() error { + ctx := context.Background() + + if !flag.Parsed() { + flag.Parse() + } + + if err := config.ReadConfig(); err != nil { + return fmt.Errorf("config.ReadConfig: %w", err) + } + + subscriptionURL, err := config.GetRequestSubscriptionURL() + if err != nil { + return fmt.Errorf("config.GetRequestSubscriptionURL: %w", err) + } + + subscriber, err := pubsub.CreateSubscriber(ctx, subscriptionURL) + if err != nil { + return fmt.Errorf("config.CreateSubscriber: %w", err) + } + + bucketURL, err := config.GetResultDataBucketURL() + if err != nil { + return fmt.Errorf("config.GetResultDataBucketURL: %w", err) + } + + logger := log.NewLogger(log.InfoLevel) + + for { + req, err := subscriber.SynchronousPull() + if err != nil { + return fmt.Errorf("subscriber.SynchronousPull: %w", err) + } + + logger.Info("Received message from subscription") + if req == nil { + // TODO(log): Previously Warn. Consider logging an error here. + logger.Info("subscription returned nil message during Receive, exiting") + break + } + + // don't process requests from jobs without metadata files, as the results will never be transferred. + // https://github.com/ossf/scorecard/issues/2307 + hasMd, err := hasMetadataFile(ctx, req, bucketURL) + if err != nil { + announceError(err, subscriber, logger) + continue + } + + if !hasMd { + // nack the message so it can be tried later, as the metadata file may not have been created yet. + subscriber.Nack() + continue + } + + exists, err := resultExists(ctx, req, bucketURL) + if err != nil { + announceError(err, subscriber, logger) + continue + } + + // Sanity check - make sure we are not re-processing an already processed request. + if exists { + logger.Info(fmt.Sprintf("Skipping already processed request: %s.", req.String())) + } else { + if err := wl.worker.Process(ctx, req, bucketURL); err != nil { + announceError(err, subscriber, logger) + continue + } + } + + wl.worker.PostProcess() + subscriber.Ack() + } + if err := subscriber.Close(); err != nil { + return fmt.Errorf("subscriber.Close: %w", err) + } + return nil +} + +func announceError(err error, subscriber pubsub.Subscriber, logger *log.Logger) { + // TODO(log): Previously Warn. Consider logging an error here. + logger.Info(fmt.Sprintf("error processing request: %v", err)) + // Nack the message so that another worker can retry. + subscriber.Nack() +} + +func resultExists(ctx context.Context, sbr *data.ScorecardBatchRequest, bucketURL string) (bool, error) { + exists, err := data.BlobExists(ctx, bucketURL, ResultFilename(sbr)) + if err != nil { + return false, fmt.Errorf("error during BlobExists: %w", err) + } + return exists, nil +} + +// ResultFilename returns the filename where the result from processing a batch request should go. +// This naming convention is used to detect duplicate requests, as well as transfer the results to BigQuery. +func ResultFilename(sbr *data.ScorecardBatchRequest) string { + shardname := fmt.Sprintf("shard-%07d", sbr.GetShardNum()) + return data.GetBlobFilename(shardname, sbr.GetJobTime().AsTime()) +} + +func hasMetadataFile(ctx context.Context, req *data.ScorecardBatchRequest, bucketURL string) (bool, error) { + filename := data.GetShardMetadataFilename(req.GetJobTime().AsTime()) + exists, err := data.BlobExists(ctx, bucketURL, filename) + if err != nil { + return false, fmt.Errorf("data.BlobExists: %w", err) + } + return exists, nil +} diff --git a/cron/worker/worker_test.go b/cron/worker/worker_test.go new file mode 100644 index 00000000000..cbe7451f207 --- /dev/null +++ b/cron/worker/worker_test.go @@ -0,0 +1,57 @@ +// Copyright 2022 Security Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package worker + +import ( + "testing" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/ossf/scorecard/v4/cron/data" +) + +func asPointer(i int32) *int32 { + return &i +} + +func TestResultFilename(t *testing.T) { + t.Parallel() + testcases := []struct { + name string + req *data.ScorecardBatchRequest + want string + }{ + { + name: "Basic", + req: &data.ScorecardBatchRequest{ + JobTime: timestamppb.New(time.Date(1979, time.October, 12, 1, 2, 3, 0, time.UTC)), + ShardNum: asPointer(42), + }, + want: "1979.10.12/010203/shard-0000042", + }, + } + + for _, testcase := range testcases { + testcase := testcase + t.Run(testcase.name, func(t *testing.T) { + t.Parallel() + got := ResultFilename(testcase.req) + if got != testcase.want { + t.Errorf("\nexpected: \n%s \ngot: \n%s", testcase.want, got) + } + }) + } +} From bd0de45a904d22741f441699c88ff258875f1387 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Oct 2022 09:19:34 -0500 Subject: [PATCH 3/3] :seedling: Bump github.com/google/go-containerregistry (#2371) Bumps [github.com/google/go-containerregistry](https://github.com/google/go-containerregistry) from 0.11.0 to 0.12.0. - [Release notes](https://github.com/google/go-containerregistry/releases) - [Changelog](https://github.com/google/go-containerregistry/blob/main/.goreleaser.yml) - [Commits](https://github.com/google/go-containerregistry/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: github.com/google/go-containerregistry dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 39 +++++++++++++++-------------- go.sum | 79 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 61 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index e4555718004..0a8cf2f28ca 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v1.2.3 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.5.8 - github.com/google/go-containerregistry v0.11.0 + github.com/google/go-cmp v0.5.9 + github.com/google/go-containerregistry v0.12.0 github.com/google/go-github/v38 v38.1.0 github.com/h2non/filetype v1.1.3 github.com/jszwec/csvutil v1.7.1 @@ -29,13 +29,13 @@ require ( github.com/shurcooL/githubv4 v0.0.0-20201206200315-234843c633fa github.com/shurcooL/graphql v0.0.0-20200928012149-18c5c3165e3a // indirect github.com/sirupsen/logrus v1.9.0 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.6.0 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f go.opencensus.io v0.23.0 gocloud.dev v0.26.0 golang.org/x/text v0.4.0 golang.org/x/tools v0.1.12 - google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc + google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -53,26 +53,27 @@ require ( require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.1 // indirect + golang.org/x/mod v0.6.0 // indirect golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect ) require ( cloud.google.com/go v0.104.0 // indirect - cloud.google.com/go/compute v1.7.0 // indirect + cloud.google.com/go/compute v1.10.0 // indirect cloud.google.com/go/iam v0.3.0 // indirect cloud.google.com/go/storage v1.23.0 // indirect - github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Microsoft/go-winio v0.6.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/aws/aws-sdk-go v1.43.31 // indirect github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect - github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect github.com/containerd/typeurl v1.0.2 // indirect - github.com/docker/cli v20.10.17+incompatible // indirect + github.com/docker/cli v20.10.20+incompatible // indirect github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/docker v20.10.17+incompatible // indirect - github.com/docker/docker-credential-helpers v0.6.4 // indirect + github.com/docker/docker v20.10.20+incompatible // indirect + github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect @@ -89,17 +90,17 @@ require ( github.com/googleapis/gax-go/v2 v2.5.1 // indirect github.com/googleapis/go-type-adapters v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect - github.com/klauspost/compress v1.15.8 // indirect + github.com/klauspost/compress v1.15.11 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/prometheus v2.5.0+incompatible // indirect github.com/rivo/uniseg v0.2.0 // indirect @@ -111,14 +112,14 @@ require ( github.com/xanzy/ssh-agent v0.3.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect + golang.org/x/crypto v0.1.0 // indirect golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 - golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 // indirect - golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/oauth2 v0.1.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.1.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/api v0.93.0 // indirect + google.golang.org/api v0.96.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.48.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 799cf8c4995..38378a79fb1 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,9 @@ cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJW cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/datacatalog v1.3.0 h1:3llKXv7cC1acsWjvWmG0NQQkYVSVgunMSfVk7h6zz8Q= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= @@ -125,8 +126,8 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= @@ -208,8 +209,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ= github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w= -github.com/containerd/stargz-snapshotter/estargz v0.12.0 h1:idtwRTLjk2erqiYhPWy2L844By8NRFYEwYHcXhoIWPM= -github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M= +github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0= +github.com/containerd/stargz-snapshotter/estargz v0.12.1/go.mod h1:12VUuCq3qPq4y8yUW+l5w3+oXV3cx2Po3KSe/SmPGqw= github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -219,7 +220,6 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -228,14 +228,14 @@ github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mz github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= -github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.20+incompatible h1:lWQbHSHUFs7KraSN2jOJK7zbMS2jNCHI4mt4xUFUVQ4= +github.com/docker/cli v20.10.20+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.0+incompatible h1:l9EaZDICImO1ngI+uTifW+ZYvvz7fKISBAKpg+MbWbY= github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.17+incompatible h1:JYCuMrWaVNophQTOrMMoSwudOVEfcegoZZrleKc1xwE= -github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o= -github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= +github.com/docker/docker v20.10.20+incompatible h1:kH9tx6XO+359d+iAkumyKDc5Q1kOwPuAUaeri48nD6E= +github.com/docker/docker v20.10.20+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= +github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -348,10 +348,11 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.11.0 h1:Xt8x1adcREjFcmDoDK8OdOsjxu90PHkGuwNP8GiHMLM= -github.com/google/go-containerregistry v0.11.0/go.mod h1:BBaYtsHPHA42uEgAvd/NejvAfPSlz281sJWqupjSxfk= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.12.0 h1:nidOEtFYlgPCRqxCKj/4c/js940HVWplCWc5ftdfdUA= +github.com/google/go-containerregistry v0.12.0/go.mod h1:sdIK+oHQO7B93xI8UweYdl887YhuIwg9vz8BSLH3+8k= github.com/google/go-github/v38 v38.1.0 h1:C6h1FkaITcBFK7gAmq4eFzt6gbhEhk7L5z6R3Uva+po= github.com/google/go-github/v38 v38.1.0/go.mod h1:cStvrz/7nFr0FoENgG6GLbp53WaelXucT+BBz/3VKx4= github.com/google/go-github/v45 v45.2.0 h1:5oRLszbrkvxDDqBCNj2hjDZMKmvexaZ1xw/FCD+K3FI= @@ -428,8 +429,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -488,9 +489,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA= -github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -588,8 +588,8 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -670,8 +670,9 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -710,6 +711,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -760,8 +763,9 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48 h1:N9Vc/rorQUDes6B9CNdIxAn5jODGj2wzfrei2x4wNj4= -golang.org/x/net v0.0.0-20220805013720-a33c5aa5df48/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -784,9 +788,9 @@ golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY= -golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -799,8 +803,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -842,7 +846,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -882,14 +885,14 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1026,8 +1029,8 @@ google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6r google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= -google.golang.org/api v0.93.0 h1:T2xt9gi0gHdxdnRkVQhT8mIvPaXKNsDNWz+L696M66M= -google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.96.0 h1:F60cuQPJq7K7FzsxMYHAUJSiXh2oKctHxBMbDygxhfM= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1131,8 +1134,8 @@ google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljW google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc h1:Nf+EdcTLHR8qDNN/KfkQL0u0ssxt9OhbaWCl5C0ucEI= -google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de h1:5ANeKFmGdtiputJJYeUVg8nTGA/1bEirx4CgzcnPSx8= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=