diff --git a/.ci-operator.yaml b/.ci-operator.yaml new file mode 100644 index 00000000000..0f596d54ae1 --- /dev/null +++ b/.ci-operator.yaml @@ -0,0 +1,4 @@ +build_root_image: + namespace: openshift + name: release + tag: golang-1.19 \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 710426cef18..81023c7e9c7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,3 +1,8 @@ +run: + deadline: 10m + skip-dirs: + - apis + linters-settings: gocyclo: min-complexity: 30 @@ -13,38 +18,55 @@ linters-settings: - dot skip-generated: false custom-order: true - + errcheck: + check-type-assertions: true + exhaustive: + check: + - switch + default-signifies-exhaustive: true + funlen: + lines: 100 + statements: 100 + ignore-comments: true + nolintlint: + allow-no-explanation: [ funlen, lll ] + require-specific: true linters: - disable-all: true - enable: - - bodyclose - - dogsled - - dupl - - errcheck - - exportloopref - - gci - - goconst - - gocritic - - gocyclo - - gofmt - - goprintffuncname - - gosimple - - govet - - ineffassign - - misspell - - nakedret - - nolintlint - - staticcheck - - stylecheck - - typecheck - - unconvert - - unparam - - unused - - whitespace - - lll + enable-all: true + disable: + - containedctx # detects struct contained context.Context field + - depguard # [replaced by gomodguard] checks if package imports are in a list of acceptable packages + - exhaustivestruct # Prevents empty struct. We use a lot of these so I think it is safe to disable. + - exhaustruct # Prevents empty struct. We use a lot of these so I think it is safe to disable.c + - forbidigo + - gochecknoglobals # Prevents use of global vars. + - gofumpt + - gomnd # Doesnot allow hardcoded numbers + - gomoddirectives # Doesnot allow replace in go mod file + - interfacer + - nestif + - nilnil + - nosnakecase # snakecase is used in a lot of places. Need to check if that is required. + - paralleltest # [too many false positives] detects missing usage of t.Parallel() method in your Go test + - tagliatelle + - varnamelen # doesnot allow shorter names like c,k etc. But golang prefers short named vars. + - wsl # [too strict and mostly code is not more readable] whitespace linter forces you to use empty lines + - wrapcheck # check if this is required. Prevents direct return of err. + + # Need to check + - nlreturn # [too strict and mostly code is not more readable] checks for a new line before return and branch statements to increase code clarity + - goerr113 # [too strict] checks the errors handling expressions + - contextcheck # Requires to pass context to all function. + + # To be fixed + - gocognit # https://github.com/opendatahub-io/opendatahub-operator/issues/709 + - cyclop # https://github.com/opendatahub-io/opendatahub-operator/issues/709 + - funlen # https://github.com/opendatahub-io/opendatahub-operator/issues/709 + - godox # https://github.com/opendatahub-io/opendatahub-operator/issues/699 issues: exclude-rules: - - path: tests/e2e/(.+)_test\.go + - path: tests/*/(.+)_test\.go linters: - - typecheck \ No newline at end of file + - typecheck + - dupl \ No newline at end of file diff --git a/Makefile b/Makefile index 34f914745ee..528cbba4287 100644 --- a/Makefile +++ b/Makefile @@ -3,16 +3,23 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -IMAGE_OWNER ?= opendatahub +ifneq ($(USER),) +IMAGE_OWNER = $(USER) +else +IMAGE_OWNER = opendatahub +endif VERSION ?= 2.4.0 # IMAGE_TAG_BASE defines the opendatahub.io namespace and part of the image name for remote images. # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both # opendatahub.io/opendatahub-operator-bundle:$VERSION and opendatahub.io/opendatahub-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= quay.io/$(IMAGE_OWNER)/opendatahub-operator +IMAGE_TAG_BASE = quay.io/$(IMAGE_OWNER)/opendatahub-operator + +# keep the name based on IMG which already used from command line +IMG_TAG = latest # Update IMG to a variable, to keep it consistent across versions for OpenShift CI -IMG ?= REPLACE_IMAGE +IMG = $(IMAGE_TAG_BASE):$(IMG_TAG) # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) @@ -88,8 +95,8 @@ E2E_TEST_FLAGS = "--skip-deletion=false" -timeout 15m # See README.md, default g # see target "image-build" IMAGE_BUILD_FLAGS = --build-arg USE_LOCAL=false -.PHONY: all -all: build +.PHONY: default +default: lint unit-test build ##@ General @@ -149,6 +156,7 @@ lint: golangci-lint ## Run golangci-lint against code. .PHONY: get-manifests get-manifests: ## Fetch components manifests from remote git repo ./get_all_manifests.sh +CLEANFILES += odh-manifests/* ##@ Build @@ -161,7 +169,7 @@ run: manifests generate fmt vet ## Run a controller from your host. go run ./main.go .PHONY: image-build -image-build: unit-test ## Build image with the manager. +image-build: # unit-test ## Build image with the manager. $(IMAGE_BUILDER) build --no-cache -f Dockerfiles/Dockerfile ${IMAGE_BUILD_FLAGS} -t $(IMG) . .PHONY: image-push @@ -177,17 +185,26 @@ ifndef ignore-not-found ignore-not-found = false endif +.PHONY: prepare +prepare: manifests kustomize manager-kustomization + +# phony target for the case of changing IMG variable +.PHONY: manager-kustomization +manager-kustomization: config/manager/kustomization.yaml.in + cd config/manager \ + && cp -f kustomization.yaml.in kustomization.yaml \ + && $(KUSTOMIZE) edit set image controller=$(IMG) + .PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. +install: prepare ## Install CRDs into the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/crd | kubectl apply -f - .PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. +uninstall: prepare ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) +deploy: prepare ## Deploy controller to the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl apply --namespace $(OPERATOR_NAMESPACE) -f - .PHONY: undeploy @@ -198,6 +215,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi LOCALBIN ?= $(shell pwd)/bin $(LOCALBIN): mkdir -p $(LOCALBIN) +CLEANFILES += $(LOCALBIN) KUSTOMIZE_INSTALL_SCRIPT ?= "https://github.com/raw/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize @@ -227,9 +245,8 @@ $(GOLANGCI_LINT): $(LOCALBIN) BUNDLE_DIR ?= "bundle" .PHONY: bundle -bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. +bundle: prepare operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) $(OPERATOR_SDK) bundle validate ./$(BUNDLE_DIR) mv bundle.Dockerfile Dockerfiles/ @@ -315,7 +332,12 @@ test: unit-test e2e-test .PHONY: unit-test unit-test: envtest KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(TEST_SRC) -v -coverprofile cover.out +CLEANFILES += cover.out .PHONY: e2e-test e2e-test: ## Run e2e tests for the controller go test ./tests/e2e/ -run ^TestOdhOperator -v --operator-namespace=${OPERATOR_NAMESPACE} ${E2E_TEST_FLAGS} + +clean: + chmod u+w -R $(LOCALBIN) # envtest makes its dir RO + rm -rf $(CLEANFILES) diff --git a/OWNERS b/OWNERS index 622262139c3..6bc58b1b309 100644 --- a/OWNERS +++ b/OWNERS @@ -1,8 +1,6 @@ approvers: - etirelli - VaishnaviHire - - LavLas reviewers: - etirelli - VaishnaviHire - - LavLas diff --git a/README.md b/README.md index 55f02c241c3..0ce9facec0d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,28 @@ This operator is the primary operator for Open Data Hub. It is responsible for e Jupyter Notebooks, Modelmesh serving, Datascience pipelines etc. The operator makes use of `DataScienceCluster` CRD to deploy and configure these applications. +### Table of contents +- [Usage](#usage) + - [Installation](#installation) +- [Dev Preview](#dev-preview) + - [Developer Guide](#developer-guide) + - [Pre-requisites](#pre-requisites) + - [Download manifests](#download-manifests) + - [Structure of `COMPONENT_MANIFESTS`](#structure-of-component_manifests) + - [Workflow](#workflow) + - [Local Storage](#local-storage) + - [Adding New Components](#adding-new-components) + - [Customizing Manifests Source](#customizing-manifests-source) + - [for local development](#for-local-development) + - [for build operator image](#for-build-operator-image) + - [Build Image](#build-image) + - [Deployment](#deployment) + - [Test with customized manifests](#test-with-customized-manifests) + - [Example DataScienceCluster](#example-datasciencecluster) + - [Run functional Tests](#run-functional-tests) + - [Run e2e Tests](#run-e2e-tests) + - [Troubleshooting](#troubleshooting) + ## Usage ### Installation @@ -27,7 +49,10 @@ and installed from source manually, see the Developer guide for further instruct EOF ``` -2. Create [DataScienceCluster](#example-datasciencecluster) CR to enable components +2. Create [DSCInitializationc](#example-dscinitialization) CR manually. + You can also use operator to create default DSCI CR by removing env variable DISABLE_DSC_CONFIG from CSV following restart operator pod. + +3. Create [DataScienceCluster](#example-datasciencecluster) CR to enable components ## Dev Preview @@ -173,7 +198,31 @@ There are 2 ways to test your changes with modification: 1. set `devFlags.ManifestsUri` field of DSCI instance during runtime: this will pull down manifests from remote git repo by using this method, it overwrites manifests and component images if images are set in the params.env file -2. [Under implementation] build operator image with local manifests +2. [Under implementation] build operator image with local manifests. + +### Example DSCInitialization + +Below is the default DSCI CR config + +```console +apiVersion: dscinitialization.opendatahub.io/v1 +kind: DSCInitialization +metadata: + name: default-dsci +spec: + applicationsNamespace: opendatahub + monitoring: + managementState: Managed + namespace: opendatahub + serviceMesh: + controlPlane: + metricsCollection: Istio + name: data-science-smcp + namespace: istio-system + managementState: Managed +``` + +Apply this example with modification for your usage. ### Example DataScienceCluster @@ -186,7 +235,7 @@ components. At a given time, ODH supports only **one** instance of the CR, which apiVersion: datasciencecluster.opendatahub.io/v1 kind: DataScienceCluster metadata: - name: example + name: default-dsc spec: components: codeflare: @@ -197,12 +246,18 @@ spec: managementState: Managed kserve: managementState: Managed + kueue: + managementState: Managed modelmeshserving: managementState: Managed ray: managementState: Managed workbenches: managementState: Managed + trustyai: + managementState: Managed + modelregistry: + managementState: Managed ``` 2. Enable only Dashboard and Workbenches diff --git a/apis/datasciencecluster/v1/datasciencecluster_types.go b/apis/datasciencecluster/v1/datasciencecluster_types.go index 120ba1baf62..12e3ad0d8e3 100644 --- a/apis/datasciencecluster/v1/datasciencecluster_types.go +++ b/apis/datasciencecluster/v1/datasciencecluster_types.go @@ -25,13 +25,15 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/components/dashboard" "github.com/opendatahub-io/opendatahub-operator/v2/components/datasciencepipelines" "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" + "github.com/opendatahub-io/opendatahub-operator/v2/components/kueue" "github.com/opendatahub-io/opendatahub-operator/v2/components/modelmeshserving" + "github.com/opendatahub-io/opendatahub-operator/v2/components/modelregistry" "github.com/opendatahub-io/opendatahub-operator/v2/components/ray" "github.com/opendatahub-io/opendatahub-operator/v2/components/trustyai" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" ) -// DataScienceCluster defines the desired state of the cluster. +// DataScienceClusterSpec defines the desired state of the cluster. type DataScienceClusterSpec struct { // Override and fine tune specific component configurations. // +operator-sdk:csv:customresourcedefinitions:type=spec,order=1 @@ -58,6 +60,9 @@ type Components struct { // Does not support enabled ModelMeshServing at the same time Kserve kserve.Kserve `json:"kserve,omitempty"` + // Kueue component configuration. + Kueue kueue.Kueue `json:"kueue,omitempty"` + // CodeFlare component configuration. // If CodeFlare Operator has been installed in the cluster, it should be uninstalled first before enabled component. CodeFlare codeflare.CodeFlare `json:"codeflare,omitempty"` @@ -67,6 +72,9 @@ type Components struct { // TrustyAI component configuration. TrustyAI trustyai.TrustyAI `json:"trustyai,omitempty"` + + // ModelRegistry component configuration. + ModelRegistry modelregistry.ModelRegistry `json:"modelregistry,omitempty"` } // DataScienceClusterStatus defines the observed state of DataScienceCluster. diff --git a/apis/datasciencecluster/v1/zz_generated.deepcopy.go b/apis/datasciencecluster/v1/zz_generated.deepcopy.go index b8e57ccaca7..9454def341b 100644 --- a/apis/datasciencecluster/v1/zz_generated.deepcopy.go +++ b/apis/datasciencecluster/v1/zz_generated.deepcopy.go @@ -35,9 +35,11 @@ func (in *Components) DeepCopyInto(out *Components) { in.ModelMeshServing.DeepCopyInto(&out.ModelMeshServing) in.DataSciencePipelines.DeepCopyInto(&out.DataSciencePipelines) in.Kserve.DeepCopyInto(&out.Kserve) + in.Kueue.DeepCopyInto(&out.Kueue) in.CodeFlare.DeepCopyInto(&out.CodeFlare) in.Ray.DeepCopyInto(&out.Ray) in.TrustyAI.DeepCopyInto(&out.TrustyAI) + in.ModelRegistry.DeepCopyInto(&out.ModelRegistry) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Components. diff --git a/apis/dscinitialization/v1/dscinitialization_types.go b/apis/dscinitialization/v1/dscinitialization_types.go index 3c5055f5c95..5f7ab7403d7 100644 --- a/apis/dscinitialization/v1/dscinitialization_types.go +++ b/apis/dscinitialization/v1/dscinitialization_types.go @@ -49,7 +49,7 @@ type DSCInitializationSpec struct { // This is not recommended to be used in production environment. // +operator-sdk:csv:customresourcedefinitions:type=spec,order=4 // +optional - DevFlags DevFlags `json:"devFlags,omitempty"` + DevFlags *DevFlags `json:"devFlags,omitempty"` } type Monitoring struct { @@ -70,7 +70,7 @@ type Monitoring struct { type DevFlags struct { // Custom manifests uri for odh-manifests // +optional - ManifestsUri string `json:"manifestsUri,omitempty"` //nolint + ManifestsUri string `json:"manifestsUri,omitempty"` } // DSCInitializationStatus defines the observed state of DSCInitialization. diff --git a/apis/dscinitialization/v1/zz_generated.deepcopy.go b/apis/dscinitialization/v1/zz_generated.deepcopy.go index d8b29a11839..800f45905ed 100644 --- a/apis/dscinitialization/v1/zz_generated.deepcopy.go +++ b/apis/dscinitialization/v1/zz_generated.deepcopy.go @@ -32,7 +32,7 @@ func (in *DSCInitialization) DeepCopyInto(out *DSCInitialization) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -91,7 +91,11 @@ func (in *DSCInitializationSpec) DeepCopyInto(out *DSCInitializationSpec) { *out = *in out.Monitoring = in.Monitoring out.ServiceMesh = in.ServiceMesh - out.DevFlags = in.DevFlags + if in.DevFlags != nil { + in, out := &in.DevFlags, &out.DevFlags + *out = new(DevFlags) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DSCInitializationSpec. diff --git a/apis/features/v1/features_types.go b/apis/features/v1/features_types.go index fbe6e4bab27..d277012c14f 100644 --- a/apis/features/v1/features_types.go +++ b/apis/features/v1/features_types.go @@ -1,6 +1,9 @@ package v1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // FeatureTracker represents a cluster-scoped resource in the Data Science Cluster, // specifically designed for monitoring and managing objects created via the internal Features API. @@ -11,13 +14,43 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // no longer required. // +kubebuilder:object:root=true // +kubebuilder:resource:scope=Cluster +// +kubebuilder:subresource:status type FeatureTracker struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec FeatureTrackerSpec `json:"spec,omitempty"` - Status FeatureTrackerStatus `json:"status,omitempty"` + + Spec FeatureTrackerSpec `json:"spec,omitempty"` + Status FeatureTrackerStatus `json:"status,omitempty"` +} + +// NewFeatureTracker instantiate FeatureTracker instance. +func NewFeatureTracker(name, appNamespace string) *FeatureTracker { + return &FeatureTracker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "features.opendatahub.io/v1", + Kind: "FeatureTracker", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: appNamespace + "-" + name, + }, + } } +type FeaturePhase string +type OwnerType string + +const ( + FeatureCreated FeaturePhase = "FeatureCreated" + PreConditions FeaturePhase = "FeaturePreConditions" + ResourceCreation FeaturePhase = "ResourceCreation" + LoadTemplateData FeaturePhase = "LoadTemplateData" + ProcessTemplates FeaturePhase = "ProcessTemplates" + ApplyManifests FeaturePhase = "ApplyManifests" + PostConditions FeaturePhase = "FeaturePostConditions" + ComponentType OwnerType = "Component" + DSCIType OwnerType = "DSCI" +) + func (s *FeatureTracker) ToOwnerReference() metav1.OwnerReference { return metav1.OwnerReference{ APIVersion: s.APIVersion, @@ -27,12 +60,22 @@ func (s *FeatureTracker) ToOwnerReference() metav1.OwnerReference { } } +// Source describes the type of object that created the related Feature to this FeatureTracker. +type Source struct { + Type OwnerType `json:"type,omitempty"` + Name string `json:"name,omitempty"` +} + // FeatureTrackerSpec defines the desired state of FeatureTracker. type FeatureTrackerSpec struct { + Source Source `json:"source,omitempty"` + AppNamespace string `json:"appNamespace,omitempty"` } // FeatureTrackerStatus defines the observed state of FeatureTracker. type FeatureTrackerStatus struct { + // +optional + Conditions *[]conditionsv1.Condition `json:"conditions,omitempty"` } // +kubebuilder:object:root=true diff --git a/apis/features/v1/zz_generated.deepcopy.go b/apis/features/v1/zz_generated.deepcopy.go index 19bc376284c..bb02e0ef5e8 100644 --- a/apis/features/v1/zz_generated.deepcopy.go +++ b/apis/features/v1/zz_generated.deepcopy.go @@ -22,6 +22,7 @@ limitations under the License. package v1 import ( + conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -31,7 +32,7 @@ func (in *FeatureTracker) DeepCopyInto(out *FeatureTracker) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureTracker. @@ -87,6 +88,7 @@ func (in *FeatureTrackerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeatureTrackerSpec) DeepCopyInto(out *FeatureTrackerSpec) { *out = *in + out.Source = in.Source } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureTrackerSpec. @@ -102,6 +104,17 @@ func (in *FeatureTrackerSpec) DeepCopy() *FeatureTrackerSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeatureTrackerStatus) DeepCopyInto(out *FeatureTrackerStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = new([]conditionsv1.Condition) + if **in != nil { + in, out := *in, *out + *out = make([]conditionsv1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureTrackerStatus. @@ -113,3 +126,18 @@ func (in *FeatureTrackerStatus) DeepCopy() *FeatureTrackerStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Source) DeepCopyInto(out *Source) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Source. +func (in *Source) DeepCopy() *Source { + if in == nil { + return nil + } + out := new(Source) + in.DeepCopyInto(out) + return out +} diff --git a/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml b/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml index 1d4b6441bff..3ec5fc3e449 100644 --- a/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml +++ b/bundle/manifests/datasciencecluster.opendatahub.io_datascienceclusters.yaml @@ -35,7 +35,7 @@ spec: metadata: type: object spec: - description: DataScienceCluster defines the desired state of the cluster. + description: DataScienceClusterSpec defines the desired state of the cluster. properties: components: description: Override and fine tune specific component configurations. @@ -262,9 +262,10 @@ spec: type: string type: object managementState: - default: Removed + default: Managed enum: - Managed + - Unmanaged - Removed pattern: ^(Managed|Unmanaged|Force|Removed)$ type: string @@ -277,6 +278,49 @@ spec: type: string type: object type: object + kueue: + description: Kueue component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: "" + description: contextDir is the relative path to + the folder containing manifests in a repository + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: "Set to one of the following values: \n - \"Managed\" + : the operator is actively managing the component and trying + to keep it active. It will only upgrade the component if + it is safe to do so \n - \"Removed\" : the operator is actively + managing the component and will not install it, or if it + is installed, the operator will try to remove it" + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object modelmeshserving: description: ModelMeshServing component configuration. Does not support enabled Kserve at the same time @@ -321,6 +365,49 @@ spec: pattern: ^(Managed|Unmanaged|Force|Removed)$ type: string type: object + modelregistry: + description: ModelRegistry component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: "" + description: contextDir is the relative path to + the folder containing manifests in a repository + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: "Set to one of the following values: \n - \"Managed\" + : the operator is actively managing the component and trying + to keep it active. It will only upgrade the component if + it is safe to do so \n - \"Removed\" : the operator is actively + managing the component and will not install it, or if it + is installed, the operator will try to remove it" + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object ray: description: Ray component configuration. properties: diff --git a/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml b/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml index f092fb3dee5..a37df4ad4a2 100644 --- a/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml +++ b/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -118,6 +118,7 @@ spec: default: Removed enum: - Managed + - Unmanaged - Removed pattern: ^(Managed|Unmanaged|Force|Removed)$ type: string diff --git a/bundle/manifests/features.opendatahub.io_featuretrackers.yaml b/bundle/manifests/features.opendatahub.io_featuretrackers.yaml index ff13ed889ef..23d3f5925c2 100644 --- a/bundle/manifests/features.opendatahub.io_featuretrackers.yaml +++ b/bundle/manifests/features.opendatahub.io_featuretrackers.yaml @@ -40,13 +40,54 @@ spec: type: object spec: description: FeatureTrackerSpec defines the desired state of FeatureTracker. + properties: + appNamespace: + type: string + source: + description: Source describes the type of object that created the + related Feature to this FeatureTracker. + properties: + name: + type: string + type: + type: string + type: object type: object status: description: FeatureTrackerStatus defines the observed state of FeatureTracker. + properties: + conditions: + items: + description: Condition represents the state of the operator's reconciliation + functionality. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: ConditionType is the state of the operator's reconciliation + functionality. + type: string + required: + - status + - type + type: object + type: array type: object type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml b/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml index ee263c57229..a61a6f34a8a 100644 --- a/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml +++ b/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml @@ -29,16 +29,31 @@ metadata: "managementState": "Managed" }, "kserve": { + "managementState": "Managed", + "serving": { + "ingressGateway": { + "certificate": { + "type": "SelfSigned" + } + }, + "managementState": "Managed", + "name": "knative-serving" + } + }, + "kueue": { "managementState": "Removed" }, "modelmeshserving": { "managementState": "Managed" }, + "modelregistry": { + "managementState": "Removed" + }, "ray": { "managementState": "Removed" }, "trustyai": { - "managementState": "Removed" + "managementState": "Managed" }, "workbenches": { "managementState": "Managed" @@ -64,6 +79,14 @@ metadata: "monitoring": { "managementState": "Managed", "namespace": "opendatahub" + }, + "serviceMesh": { + "controlPlane": { + "metricsCollection": "Istio", + "name": "data-science-smcp", + "namespace": "istio-system" + }, + "managementState": "Managed" } } }, @@ -88,49 +111,6 @@ metadata: containerImage: quay.io/opendatahub/opendatahub-operator:v2.1.0 createdAt: "2023-8-23T00:00:00Z" olm.skipRange: '>=1.0.0 <2.0.0' - operatorframework.io/initialization-resource: |- - { - "apiVersion": "datasciencecluster.opendatahub.io/v1", - "kind": "DataScienceCluster", - "metadata": { - "name": "default", - "labels": { - "app.kubernetes.io/name": "datasciencecluster", - "app.kubernetes.io/instance": "default", - "app.kubernetes.io/part-of": "opendatahub-operator", - "app.kubernetes.io/managed-by": "kustomize", - "app.kubernetes.io/created-by": "opendatahub-operator" - } - }, - "spec": { - "components": { - "codeflare": { - "managementState": "Removed" - }, - "dashboard": { - "managementState": "Managed" - }, - "datasciencepipelines": { - "managementState": "Managed" - }, - "kserve": { - "managementState": "Removed" - }, - "modelmeshserving": { - "managementState": "Managed" - }, - "ray": { - "managementState": "Removed" - }, - "workbenches": { - "managementState": "Managed" - }, - "trustyai": { - "managementState": "Removed" - } - } - } - } operators.operatorframework.io/builder: operator-sdk-v1.24.1 operators.operatorframework.io/internal-objects: '[dscinitialization.opendatahub.io]' operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 @@ -200,9 +180,9 @@ spec: is the Controller for managing ModelMesh, a general-purpose model serving management/routing layer\n* Distributed Workloads(Incubation) - Stack built to make managing distributed compute infrastructure in the cloud easy and intuitive for Data Scientists. This - stack consists of two components \n Codeflare - and KubeRay.\n* Kserve (Incubation) - Kserve is the Controller for for serving - machine learning (ML) models on arbitrary frameworks" + stack consists of three components \n Codeflare + , KubeRay and Kueue.\n* Kserve - Kserve is the Controller for for serving machine + learning (ML) models on arbitrary frameworks" displayName: Open Data Hub Operator icon: - base64data: iVBORw0KGgoAAAANSUhEUgAAAUIAAAEiCAYAAACMWdvGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAOxAAADsQBlSsOGwAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAACAASURBVHic7N15eFTV+Qfw73vuTBKyQdhVkIC4orjUBREtO6KE1ewBqVqsW0Xb+lOxNq11a+tWbVW0ikAWiIAQQSEo1gWtuyAKsiS4IMoSkplsM3PP+/sjC5NJcu8kmSQQ3s/z5HmYc8895wyTvHPvPRsghBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQnQR1dAPEsWnWluJzoTECzB4itf6ls7rtDMyz5p8cftDtmgXCZcTKy8Rrd1RF52Vmku6INovOSwKhaFeJWzisCxe/CEaaX7LJzA8uGtrjj7UJSzNLunvD1BsAzvE/n8Gvd4+OmXrFb6mqvdosOj/V0Q0Qx5YufOjBgCAIAAYR3TNzU/Hs2gRPOD2BgCAIAAS6vNjlvquNmymOMRIIRbuZXVgYAebfNHWciG8DgBczOYKYEpssiGhWGzRPHMMkEHZi5eU7+u/lvVEd3Y5avvLu/QBEWmQ5DQAiHBW9AIQ3nY1PDGnDxDFPAmEnwsyq1Lvr0hLPrn+Vegr3+hzGt5HeipJSb+HKgxU7Ozx4kPY6bbKEAYAG2f1eyu+tCClHRzdAtF6xp+hcAzq91FuURKD+AT1gBhiTHYY6u5S//0Us9TvQMa0U4sglgfAoxczKVVU4CYp+B/BlANkNARjAHs9tAO5plwYKcRSRQHiUYf6ui9vrm+XyFt0ORac051wi9cu2apcQRzMJhEcJZlYub9Esl893P4DjW1iKL6SNEqKTkEB4FHB5i0aVeoseIeBccCsKIqwLWaOE6EQkEB7BDlbsPNHhUE8w89TWTwGiLysdZU+GoFlCdDoyDOEIVeLZNcthqE1gTG1dSeRm4BntpMt60xB3aFonROciV4RHGBdv78U+xzNgTG9FMZVgWs+k86qc5cslAAphTQLhEeSQd9d49tJCAH1acj4DnxPwhNfpW96DTi4NcfOE6LQkEB4hXJ7COcz8L7ToM+H3oPFwbPjAV4moNd0pQhyTJBB2MObt4S6v42kGftXsVdEYaxT0ndHhJ21uaf17OD+yqkzFal0VC8OIVT42lIN9Xu2scpoo9+nyAyd1TyppaflCHA0kEHYgF+/o7fKq5QAuaeapmzXx77qFDSoI9oQ9nB9Z5eYLwPpiKJwKplMADK5y696AhiID0AAUQWuCARPaAJQRjkLXylIA3zGhiIBNpPkjr8P86OTIGd83s91CHJEkEHYQNxf21V68AeCMZpxWSsAd0c7454nItMu8u2zFLzSrGQBGV7n1eQCcIEILxiLGAhhCjCEArmQiOEwHCl0rfyDQa8x4NTyGCo6nhPJmlyzEEUACYQdw864+2ov1aF4Q3MjavDo2YvAOq0xFruWnazauJYXpWmNg61pq6wQGXwfCdVVurihyrVwNmPfGx0z/uo3rFSKkJBC2sxZcCVaCkBnjiP9HU1eBG3iDY6C7dLIG38ig0USgVs1AaZkuDFxFMC4vKsv/ZXxUwqft3gIhWkgCYTty8Y7e2kvvADw4qBOIvtdMU7s5B3zS2GHmTLW77NxUuEszGRhsu/5MO2AgGlo/ieY/9xSiw8jMknbCXBjBXmNF0EEQ+MBw4MJuYY0HwV2lryQUuc/9nBmLGQi2zPZy8e5Dr8Z1dCOECJZcEbYDZqZSb9ECAoYHdwYtinHyHKL4ysAj35Xnn+Az9b8ATAlxM0OJtIMsltoX4sgigbAdlHoK7yOi5CCzPxvjHHBD4MBoZqbd7vwbfKZ+ENW9uG2hFIAJAAzEUMt/P3YOjL5yb+iaJUTbkkDYxlxVuxKZaF4weQl4PNoZf3tgECwsXtGtyL1qAUJxFUgoIo13taIvSGO7Ir0dpuPHAd0mFQdm3cJLw2JKnP1NA/2J1SBNOI+ILgDz2Wh6cyVm8J2tbqcQ7UgCYRsqL9/R30f0bHC5+ZGYsEG/D0wtdK84B6xeBnBSC5tRxYT1pHmZYdDaE6Om7An2xCGU5AGws+bnLQAvAEAhb4igctco1jyZgEkM9AMAAr5n4A+DYqa+3MK2CtEhJBC2kZoVpRcACKbTYEFsY0GwdNUUMOcA6NLc+gn8JYP+rb1V2aGeIjeQRlUCeK3m54bt5cv6KQpTgyI++Z4oU4eyLiHagwTCNuLyFf0fgNFBZH0nxulrsOl5UemqazXxMy14TvcmoO+Lj5n2VjPPazGZaieOdhII24Dbs+tszfhzEFm3sdMxlWhglX9iYemq/2PiB6k5qzAQfahAdw+ITnijue0V4lgngTDEmJlc3qInAdhsZk5uaHNyV+p/0D+1yLXqRgY/1IwqS0G4Nz6q8imiJNv5x7UymdW3m0viTdKnAziRwX1AFEGMrly9/MKh6h/+ltjYobly66Kz+5Y1o11HvKVL2ajaWTpQEQ1hpuMI3JW06sqAArEXxG4CDjBUkUnGN7Pu7PJtR7dZtA0JhCFW6tmdRoRLg8h6a2zESd/4JxS5V6Yzc3P2FVnrMNS1/SMTfggm86wtxeey5isU6Je7viy+GITow0erF2NoODOPwKRBFOaZtelAToSHbp1/fvejclmuZ59lZ3Sx+2JoTABorHeHe6iCigDXXnoT2L/DnmtnKjIM9iHrAZcbwHsg3qBM442UeZGfyPqPnYMEwhDax1tjyMcP287zJayIdca/4J+0y50/nlm/iOBm+2gAf4mP/uw+u86Ja7buO97nVXMASofmwQSAWzYROQxEV1eG45TEpXxpXpL96jdHisUPuc8izXNwwJ0BoFt1aov+D6IBTADTBK00sh9078h+wLWIyLcw9a64opA1WLQ7CYQhFO4JvweEEywzEX6Aw/lr/6RdFasHkM+XDdvbaYAANxQlxUdNfs0q38zN+08nUvf4vEgMptxmuLjL6QevArAkhGW2iez73eOY+E/Q3Fbzngcz8Gdmx72LH3AthaIHM+6MbvEiuaLjSCAMERfv6A0vbra9GNQ8N4b6Hah9vZ3XhJPLmwdCD/ta6CCYJsVHJbzfVI6Mr/Ydp0zjYQDp4LaZS05Ml+IIDoQ5D5RcpKEeZPCodqrSICAVmlOyHnAt9Wnjd1ffExnU4wpxZJBAGCLsMW4DIdIm25sx4YPqDTY23N5HQLggiCp+ZOZRA2Mnb2u8AUyzNhffBBP3o+2m4NVW5m3b8lvm2UyOjAlz36+B36JjFhQhAMkOZV6R/YA70zE46omko+gRwrFMVp8JgYO8sysIDcYC1sc+xXquf0qha9UvCbgxiCpKQPqKQbFTGg2Cs7f83HfWl8WvgfAk2jwIAiBY3pZ3hKy/llwYHe7+nIG56Pjf6xgGP+Ld4S5Y+jd33w5uiwhCR//CdApOn3Ez6h7CN45A//LfZKmQN0QA/CzsxwpWaFaTBkZP+7yxg7O/3H+BNh2fAJjQ3Ha30NKXzuqxrp3qCkr2/e4MKPVfME7u6LYEGOX18edZD7ku6+iGCGtya9xKzFvCXF6+1SZbJTlRb2wguUvnMXBqEBXcdFJswruNHcrYVDxVM2eDmj8Fz48H4E3MtF8pcmlwsWIyGNwHQD8CHcfgngCKALzw7f64v7WirpBiZsp5oOx+Jr6rNeUQ8AMD3wD4EQw3FJdBUxQIcSD0B+MUAN1bWHwfaKzLftCVnnZXzLLWtFO0HQmEreT2dJkMQi+rPAz+TzQNqluWalfF6gHs8/3BrmwCPR8fO+XFxo7N/PJgCjEvQvM/Q2bgv8RYBkN/WIEen+cNIU8zyzgiZD/kegREtzX7RMZ+EPIArPeB/3v13bEH7E7JebA4ntkYpUGXEzAZQEQzagxnxpLFD5TekHF37HPNbq9ocxIIW4mhrrYZk+bVJv3DP4FM371oehmrWls4OuaWxg5kbCqeSsyLARjNaOpegJ5RPry04Nyjf8xb9gPuh5m5uUHwPQL/w9UzZvX111OzOnxqxgm+CODFpQ8d7OrVzjQAvwcwKMgiDAI9k32/qyRtXszS5jVbtLWO3+TiKFa9ERN/B5DFFwotjg2Ln1n7amfp8lOIjC02iyloYr40PnbqxsADs784MEwrehPBr0hTzsyPVhrmw3lDeruDPKdNzNy8/3SC+soqz8KzulPWXysGQPmKQlj1JjDmps+L2RDCMrEhkx17nK4MED0EoE+Qp3mY6YqMedEyJ/wIIleEraB9nGEdBAEi/Vz918a9Qawo82xjQTB18099NGg5gg+Ca2DqOYvO6Wk7pm0DZzrCD5T0gabeBqEswlQ/nN33H0f73GIPQPOO90Q9PiqTfKEuvKbMBS9mFr8S5nT8HYTrgjgtjIiXLnyo4lyZu3zkkCvCVij1FH0E8PkWWXbFOOMH185HLXSv7gv27QYQZnFOMXx60MC4aYf8EzOZ1a4tB9eBaUwQTTPB9OdBZ3W7P5Oo0Sl4H/78h74+8k0m1lMYOA+g3mg4isAFYDsDrzHRK5f0fOwTasVGoe17RUiFSlNS6j1RH7eunOBl3e9KAeE5wH8Od5Ped/eI/mVzb9FF25ArwhYq5W094eXzrPIQ4SX/SfnMvl+TdRAEGH8PDIIAsOvLQzcBQQXBMiaeseis7msbO/jBz7eO0KD7fPBeBoZi6+/CGADnEXAeMc97f9/c79/fR48WH/L9+4qTn6yyOrGDfUaMian3RP3UnpWmz4vJzXqwbDtYrwHQ2yb7xdH73X8EcG87NE3YkHGELcSesHGw/v9jnw8L617wUoOYr7Updl9VFZ4KTLxm677jAb4viGZ5wEhcdGaPBkHwgwO3nPHeT3NXatA7AEbatL0p/Zj50W5dja3v75ubzjZRtGPwR07lHZU2L7pdg2Ct9LuiPjHAlwKwr59wR+4Dpae0fauEHQmELURkO4B5c1yXgUW1LwrLuowB0QDrU/ip03pNcQWmer3G3wB0tanPhOL0hUO7N5j18f7Pt12tTeNTIky2KSNY8cxY/P6+uS9/sff3USEqs/UI25ygK5Pu7NhlwlLujv1Gs7oC1bsCWgk3QQ2++ET7k0DYAsxMIIyzzkT1rsqIebpldsBnKHo+MH3W5oNDCUgNolF3LBzS4+X6SaD3fpqbyeAFsB+u0xLTy5Rv4/s/zo1vg7KbicoMhWlJd8fs6+iWAMDMeVGfkuYM2K/3NS7rfld7LQ4hmiCBsAVcVbtOBuN4qzxKmXXT0JgzFcAJlvmBFY3uMMf0R9h8Tgy8Neis7o8Hpr+/b+4CIvzJ6twQGMoGNr578NYT27gea4wbU/4v5usObUOAtHti8xkNH3U0QLinHZojLEggbAEiOsfyOFAe5dDv1L4uKj3vQsA6cDJTVmDarC0HTwTxVJvmlJoGzQ7sHd748213Aphlc26oHKd8tLIDb5PXpc+LXmifrf1pZ9ldAL6zyTY654GSi9qjPaJxEghbgIGzbY5/RHRyXa8qke1uduXhMVTQIFXjetj17DMeyD4jbrd/0vv7bp0M8P02dTbFheoVsJvrnHJlvtQBHSheA9zoDJwjwaw/9C1j4Ha7fJrVNe3RHtE4CYQtYntF+IX/aya2+7Z//XhKKK+XwkywfzZ40BFm/ts/YcPPN0Yz03wE/9luZvCd0DQ0zFceNbz347Hf9/ohDIr6gSgdwMsAglpTj8EzPth3W2KQ9YYGUVbK3bHf2GfsOOl3RS8D8JllJkLy0ke5NYtniFaQcYQtoXC21SNwBn0RkHShVXEEfjMwbeZXBy4E1ECr85jpiRdO61WvlzmCwn7HHNR0r5+Y+ffDe8dlB+57kkR5JoAfAGQDyP7gwC1nmKbxOMGmgwgAg+//mOesOJ/mt8dAYTYUHzGr4TSFiDj7QfffmTnbIltXX6VrAoBX2qtd4jC5Imymfbw1Bmy9L4mGuan23zsr8k8EYLk4p6nof4FpZBrjbZriDTNUvQfxH/x0ax9m/N7mPAD8mcNQF1zS54nFdps/AcCwHk9+NbxXt8sR3Dajgz37IucEka/VCHj3SOsgaYqjKmoZAMtVbhg0tp2aIwJIIGymsCqn7YrD5c4uddPIyNR2aw5WVkZWbWqQSmw9i4T47f8M6VpvT2Qmug4207sY+MYM84y5sMejdg/w61dHmXp47yfuItDDQWSfa5+l9ZiwuD3qCYWkTPJw9WMGK8HMHBJtQAJhM5FBdoHw0PF0fN3zPmKyvL0FsHUIJdVbDzBxKRuA9T4mxGpVYBoDU2zqqiJlTLm029PFNvmaNKxX17vBaHArH2Dwe3t/e2ZL6wiWMo0jbssAK0rzapsspy38u8tuap5oAxIIm4m07fO3vfVeEewC4a7AhPBT9w8GrDeCMphe9X+9cf9tJwCwWgACxPzU8J6PbLVpjyWiTA2mubDrWVZkN+yntXak3hPZrKvajuZw+N6GTceTYeK0dmqO8COBsJk0WT/vQ8AcU2bub5WZGUWBaUo5zrCpw/XC0G71AigxT4T1akImAX+3KTcow/s+thmgNVZ5CHRFKOqy0G6ryoRKzdS/xnchrMUSCDuCBMJmUkTW+w8T/VzvZfUKLlYF/thIaj/LKoDtgWnMtislbxzW54kQLkSg7Xo3z2zTMYVsE1COXJZDfRg0uL0aIg6TQNhMWpP18vjM9cYDMthytgUxN7L4KVtedTKwo0E5NucA/J718eYxtGn3nDDmnb239Axlnf6YqLCtym5TxJbtJs1x7dUUcZgEwjZGIMtAyNQwEDLYpueX9zZMI8spfARqOI+5FXb3+elb2Cwo4DAcdivmtBxpu5VdjkyarNtNNncQok1IIGxjxNaD1hXQcAl5Issd0gjU4JaTAcuAq0Eh3a+kZtC15fL32qy/4o1WDdvdUgTVYLmyowGRXSDk2HZqivAjgbCNMaHc6rjWDXuHia17Fokbfm4KZPn8z/7WuXk+OHBLLACnZaZwXS9YGabtmopI3MLWK3jXUCY1WMX7aMA2ny3Y5tGLaBMSCEOMwPWvABmWGyARNTpMxvLqTauGAUiDLW99CXyy1fFmM9VQmxxalan6awPadTQBiPD9ZB1ca4ty6qN9YylxBJFA2EykYH0lwlRvs3ci60AIUGMPxy3rIEZ8gzSgsd7nw80CXVm9LmJoaFJ2q11/O7z/YxX12sBk23nSLbJPyHebE8KOBMJmYm29FwVT/QHXDFivmNzI8v0MshsofHqDFK0/tDmn9wf7DtltLxCUNdtvCQc4ySoPMxrrpbZbvNV8cjA8NnmECDkJhM1kKNN6LF7Ayi/MsB7mwQ1nnrDmIptm9Ltm6756vYthfSrfBshy6hwDD4XiqjCum+NGMKz3X6HGVlHhi61PwRfw2/WvLbiqdp/h9u4ce6iiyG7GjziGSCBsLpsrQhB6MfPh/1ebcWNo5NmdQ9NmWA9NIY9XDfNPOJ/me5nZcrYHgKHv7y9u1bLwHxy45QxmzrTJVsUUVm/PlsSlbBDBMhACaLO5w4cqdw8q9RS+x6S3aFYFyuBdpd7CVaW8rc3GOoqjhwTCZuIwtpudYZRU7q67WjKUXSDECd+Wraw3BnDBuXGH0MgcZH8KNK2RxBybugCmzPf3zU23zdeIjftvO0GbxkoAlkM8CPTqiF5/q9dj3GXIoaGw2YmPNL9qdbyl9vLeKKV0AYDh9Q4wEuB1vlLvi0sck+QXoJmicdJ+2IzJU4rrelR1ZLctgPVzL9NUDRZuJeJ3Gstbdxw0PTPgD/iSXo+vBvC+1XkAiBmL3vtpbuZSTgx6qMbGn347HJo/AmA3Bcw0zYYbRpGGXfA9VLat+0fBtqc5orwVM4GmpiDSJSW+QlkH8BgngbCZiEgDvNkyjzq8p8lAGlUJkE1+PTwwTYMabNLuj8F9dn5VMjIwXYHvsDqvtkoi/Knf/hO+2PjT3KkbOLPJQd//23f7KRv3zV0IUu8AOC6Isl8acdxjW/wTbtnO4Qy+2vo0fjMviYLaEqC52GbrVQX6RVvUK44eslR/y2wCmn7exeCh9V/r/5HFHxszTQJQL4CFkbHOy6YHQJMDjMk05wH11wYc1vuJdzf+PPdlAFdZvoPqhg0BYUX4vpLi93++da0GthCpnwGOZMYgYhpjsrZbCcdfiTLMBleDpRXFV4Fg/SyOaZ3l8dbQuNhq+QdmPipnqYjQkSvCFgjcnKkBpnP9XyqijdYl8uk7S/LrdZr8Z0jXgyCbzgOi0VdvKr4sMFlT2DUAvrSus179cQxKIdB9YH4WjMcIuAXEzQmCGsQZw3o8+b1/4sgN7GDC/9mcW+E0jLxm1BW0QxVFA0F2V7LUJrfk4ughgbAFmFTDpfXrG3SocnfdMymHNl9nm3m5iswZgWla0wK7tmjiP9fseFdnRK+/uQztm8zAfrvzQ4WJ7hje64kGnR39ex26GcBZ1mdTTuC2A6FCSl9ik6Uy1lluvcOc6PQkELZAlaNiE8CWgc1Qum7zpX6x0w8QGh1gfBjRdYFj/Kq2dstHI0tu1TsNGDlrc/FNgekX9X2qUDGmwmbDoJAg+vslvR57JDB55heu3sTc4FY5EIOebpuGASBq8Pw1IMPHRENkEPcxTgJhC/Si01yA+sAqDxPq7UJHhHybYk/6tuwXo/wT8pLIZMLjtg0i/H3mFwcb7BFycZ/H3zOZLgRhS2OnhUAVAb8a3uuxBh00iUvZIMO7AEA3mzLeW3RWtzZZbZqrr5Qn2WQK6TqN4ugkgbCFiPC6ZQbGGOaP6xYQ8CleCpv9KjT41sC0buFxzwM2s1OACFLIDpxtAgCX9nlsl1LmcALnwmb9wGbaDtajL+79+ILGDnY5vfghMCbalMFKcxDbj7ZMqbfoIgIst0qA/XAjcQyQQNhCmk27Xs7YUrNn3VXh4Mip37Ft8OSEnWWr6u1e9+TJVEWEeUE06Syf11gz84u9DdYlHNbjydKLez+RyoovAvBWEGU13URgP4PvPFRinjW8zz8b7QS6elPxrwD7/ZUZyF5wdg/LK+vWIKYGz10DVJSHdVnfVvWLo4cEwhaKdQ76BDYLKhDrX9VL0PysXbkG872BaS8NicslUEEQzRpBKix/zsd7Gt0B75KeT3w0vPfjoxTzeAaeB2ymC9ZieACsI8aNHngGXtL7iYevOPnJqsayzvzy4E1M/HwQpZY7wHcFVX9LEU+3Po61famvLOclZBxhSxGRLvUUvQ7wzCYzMSWU8raesXTqfgAYGONZU+QK293YijN1pzAm7XbnjxkQnfCGX2VMnxXPYQc2wW4zKGBUZXjE+l9tPpD84lk9Gl3FZlifJwoAFDBnqg9+PnQxAxcz4VQi9AIQBYYXzMVEahdDbwpzRqw7v/vDJZa1MtPMzQf/Qozg5jIT3/rimY23LxRc3sKRdhtaMfPytqpfHF0kELaCJr1IMTUdCIEw9jpTATwJAERJZqFr5UMALHtJNeunC3nD0OpZKdUWnBtXdPWXB69nRnYQTbvYBH02a/PB2QvP6t7k/F2iTI3q3uxWdRjM+nz/Cfiy+BkQWXdM1GAga9GZPYK5amwxBn5rk8XLTqNN5jaLo4/cGrdCV8fANwj41ioPgX7jP6m/PLrqBTDvtin6ZLhdDW4bXzqzew4ITwXZvB4AVs3adHD+NVv3WW7s1GLMNHPzgWthqC9h1zt72NZK5fuNVQbTEUSnjslNdjwdrNh5IpgTLM9nrO9GAyyXLRPHDgmErUBEWjMttMl2httTWPesaggleRh0v13ZDL57V+krlwamf7sv7jaC7VCcuiaC8Guf19g+68uDD167paR7kOdZmvMxO2dtOpg2a3PxJwR6HvZDZKox9mhlJOQN6W25aEVFZcR+AF6LLF7lqWxyawKHUjcCZHe3M9/muDiGSCBsJWa1ADbDUjTRPPab/bE7JvZFAJ9bnUOAQxFlbyvNrzdH961R5AuvqkwhguXqNAEiwbjTq80fZm0qzsvYVDz1lu0cbn+aH2aavaX4nFmbD95dGV68A4QsEM61P7EagX5S0GMWD+lqOUAcAK7PpHIASy1KW5yU2XgwLeHvuoNwvU0Vu2PC4oP9MhHHAHlG2ErdIk7cWeIpeoPATS7lRMA5rqrCSai5khtFo3xFpfk3Mul3YfFlxEC/cNI5W3jplUMoqW72w/zzjy+f+cXeiUqFr2Cw5coqASJAfJUCriqpLC6dtfngJ8z0uVL4XGvsViC3Jl+5kx1VptLHaaA/MfoDGEpfHhqnwX1sa2jcPq157KKze24N9gQf+FYH6HQA5/mnE/CO9lTNbeo88ph3gWyuUAn/JmqblW7E0Slk+8wey1zeotHM/IZVHgY+j3XGn+//B1joWvUMwHZXLwCQGx/9WXpN50adW7ZzeEll8fMAMlrW8naxycFq2gtDu1kuNNuYpZkc5nG6MhThMobyMXHBjqrovMxM0o3lLy/f0d/nML4BYLUvdAWczv6x1K/tpx42Iut+960gtpottC797piQ7C0jgidXhCEQ44x/s9RTuBGBKyD7IeCcUm/hDcDhzg5lqru0YU6E/aZGKYWuc/cA+J1/4pMnUxWAmbM2HfgMRA/jyPs8l7D2XPvC2S0bq5eUSR4AL9T82PI5jExYB0EAWNRRQVAcueQZYYiwsu8AIdD9ZVxUtyTUgG6TilnrRNisYA0ARLi90LXy6cY2X1o4tMejivRwAEHferaxUoB+u/DMuNRFLQyCzVXiKRwGwGbxV1T6TG37ObUp4jbdnEq0jATCEOnqiF8DkN3iAbGml+v9IQ7qOu1DJgpmCh0A/Ga369yXPuZnG2yCvuDMnh9VuMrOI+K/AKho5Nz2wchxOM3TF54V92RjO9J9x0u7FJXmX1xYsmrYdl7TvA6bxWnmTgAAIABJREFUpqrk7eEE/AeA9dYDTE9273KS5XCnNsf42fI48d52aonwI4EwhDTpu4PINrvEV3SFf8LAqIRHAAQ1y4EJGT3dfd8sKlvWYLHRvOH9K146s8eflI/OAPhFWA9BCSkivMNKjVk4tHvaC6f1ajC0hZmp0LXyBp87/AcmvRGK33e6vd/vcq9s9fNNl8eRCcBuEdlDHGY81Nq6WstHXACgyRWxCcp+Ay4RctJZEmKl3qI8MNstk7/PcNLZURT/Y23Cd7y0i+kOX8fAiCCr+pGZkwfFTm1yGE3aV8UDHCbfjOpbxl5BltscHgBLidXjLw3t9klTmbaV5vcMU/xCE4OcmRlTB8VOWdWSBpR4dl5AoI224wYJd8c6Bz7YkjpCbfH97jQiXojAK1jG/PR5McF0nokQk0AYYjU9l18DaLAKjD8GrY91DphQvRlUte9Klnb3qfB3YH91U8sE0z/DY+ie4ymhvKlMiVs4rIt5aCIpPbVmf5TW7OVbRaC3mfgVRb7lC4b0tryVK3SvvByM5wGcYJHt84ExU4Iek1irlLf1hDfsY8Bms3nQjhinMZSof8c9Mgiw+K8lw0ip2wGcBuAnBi1OvytqIbXxBveicRII20Cpt/BOMOyvPojmxTrjH/BP2lH+Sn/DpDcANNj43cIuUnRzfNRk+w3SmSn9q4OnK42LAQwl0KmoDiQ9an5qH5e4Ub26zh6AtjHxNjbxviO25KMFAwdWNl74YTsr8k9UPv0YAOsVYGpadSB6b/j5dH3Qt/LMbJR6d79uNX6zhibiUTHOQW8HW7Y49kggbAPMW8Jc3shPAQyxyaqJOSUmfFC9jYsK3av7gn1rAQxt4rym/JdY3RUfm9Bhi43ucC3vbZDxOwA3ga2viv2Ux0dPjm7O1VBpVeHDINhuXUqgp2LC4m8JtlxxbJJA2EbcVTvP0qQ+hP24tkoQjYl1xtdb5HT3oVfjtGGuATCsuXUz+A1i46n4mIp8oqR2mUGxs3T5KYqMGwH8GkCj6yE2jbIHxky22wC+Tqmn6BqAn4f9729hpbN8aG8aYjm3WQgJhG3I5Sm6kcH/ss9J+1n7Lu4aMbjePNw9nB9Z6dbzCQg6SNQvFkXMlKUUL4uPmhLyndq+L13ew6uMSWBci+pOnpb8Pn3rMNTw/pEJPwST2VW1K4mJsmE3VAYwiTA2xjnwrRa0SRxjJBC2sVJv4QpU7yZnjfEjQY2NCR/wVeChXa6VNxPwCCw2ew/CLoAKmPhd8up3B8ZNK2puAd+XLu/hUc5fEJvDAJoI4ALYByQra5lw9aDoKUGtlH3Iu2ucYsoHYD/+kPiOWOegv7eibeIYIoGwjZXwd93J6/sUtj2bsAyGRaWvDGeihQBOClXTANoB8HZm/EDEpSAqAVMFCJHQCCfiaGacCEI/EAaBER+iuj1MNG9gVMIjwT4XLPEWXk6M5QC62GYmejnGMSBJemBFsCQQtgNX1e4hTPo9AF2DyL5PgcdFhw36IvDAHs6PrHTp+4hwK1p3JdaB6BOlzOsHRE1rctxhoFJP4dUAngPQYEZNI76qclYNq95yVYjgSCBsJ27v7jGa9WsI7o+5FEqnxzpOanQp+cKSVy6Con8jYImqI1wxgHnx0VXzm9OBU+otuhvMf0VQv6u0H9q8JDbipG9a3kxxLJJA2I5KPYWzUb2SSjD/7xqEe2Ic8Q81dovHzFTozk8k8H0ATglxU0OpAqDnfHD89eSYKyx3/fPH/F0Xl9f3FIBrgjylVEOP6RZ2UptsFi86NwmE7czlLZrH1Vc4QWFgSZWz/LqmhoBs4A2OgS7X1SA9l0Fnhq6lrUOAG4SnNfBIsJ0htVxVRaczYSnAQb0fAsqZ+PJY56DmrNotRB0JhB0g6JknhxUS4Rq7oSBFrlWjGXwzgMnouGeInxL4BfZx1sC4aYeae3LNVfNTsJmi6MfDhCldnQNfb25dQtSSQNhBSjyFt9UMiQn2M9AEPOlyht99PB3f5LxioHqRAyfMqQSaAcJotG7YjS0Cfwmi10DIael4xeKKwnhD8VMgujL4elHOSic39SxViGBJIOxALk/hDQz8C836HGgHGHfEhsevCCb3zoNLu8IRcalSuISYRzBwPuxnu1gxCfw1M30Coo3aQa+f1CWhxWv8MX/sdPl63EaMP3GzZqTQfgYndA0b+EFL6xailgTCDuaq2nUVEy1A8LeCtd7W0L9rbucA81Lj25LIAdrhO5kYg5nQH0xxALoB3I1Aihk+IrgYMAHsJcJ3zPwDtNrdpSJ8c9++E1q96jQzk8uzeyqI74P9nOxAhdA8MTZi0LbWtkMIQALhEcFdtWuoJnoFwMBmnsoMLNGgv8WFxYd8Cl1bYGZyVRUmsKJMQvDbgfr5yHDSFP+1HIVoLQmER4iatfWWAhjVwiLeZEWPxhoD1hyJMyr28t6oSG9lCoNvamEAZAL+Ge0sv4NoiO0eL0I0hwTCIwjzBofLM+DP1ctL2ay43LRviJDFps49EgYWuz2F55jgXxMoA0BsC4spBuNXseEDV4aybULUkkB4BCrxFF5EwAJUr17cYgR8ysS5SvPrUWGDvmyPK0Xmj51uX89LmXkSQAkAD25difSuaeqMuC6DdoemhUI0JIHwCFUzs+J+ALciJJts0X4Qv83MbzH4fa/Tuy0U83FL+fse7PNeoIALWdMFIB4BoFvr24uDAN8Z4xz4/JF4qy86FwmER7iaq8NHYbF5fEsx8B1A2wC9TRF+0oxDYBTDUMUGm1UAYEIZSnP1LS2pXszcD0T9AR6A6o3p7VfVaTZaTE7f72JosPXWl0KEiATCowAzk9tTOIOJHkLoluE6En1ARHfHOOM3dHRDxLFFAuFRhHlLWKk38iZi3AFC345uTwh9yIoyuzri7TefEqINSCA8CjFvD3d5HWkA3R7swgRHJn6PlXqwqyN+dUe3RBzbJBAexZiZSn1F40ljLgjjcHQs1nqIQIvBvmdjwgd/2dGNEQKQQNhpuHlXH9OLJAKlALgYR9Zn62HQ2wTOdjvDl9gtGiFEezuS/lhEiBRX7BpgONRV0DwahEsBxHRAMw4A9BqxzveG8drudFJJB7RBiKBIIOzkmDc4Sr3xvyDCSGhcBsIQVA97CeFnzz6AvgL4IwJ9ZII+6uoc8AURtcueykK0lgTCY9Ae3hPZxes51cF8KhROZcZxDMQpII5BcQDHgREJQgUAMFCiAM1AFQPfE/h7Bu0mxrea9PeVzqiv+1LfVq9II4QQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEO3jmFiqf8OGDRE+n+98Zu6vlIph5v1KqaJ33nnn88zMTG117tq1a/sbhnGqVR4iOlheXr41ISGh0d3ZXn311bjw8PDhzW23w+H436hRo/YDwJo1a3qFhYWdXXvM6/XumDhxYlFT52ZmZqoRI0aMrn2ttT40fvz4j+3qnJCY2D1C03lWeUxN+xzcZfcrryw4FNQbCZAwI3WIgnlcXdtg/Ji/LGdLYL4piYlnsGkMbEkdBod9sGLFwgOB6ZOmpl1oGL7YuroV78zPyytsSR2JiYlGlWlcXvvaJO1bvWzpWqtzJk9PudL/9arluXV7Ok+ZktIfDl33u6ZhfJ+/LGdrU2VNnJ7eL4y8p9W+9mn6YfWKJV/Xvp40LfViQ5lRVu1hRkWVwV+vzcs7aJWvs3N0dAPa0htvvHGe1vpun883EUAkEYGZAQBaa1xyySX7CwoKchwOx99GjRr1fWNlKKWmMPOTVvUwMyIiInwFBQVvaq0fnjBhwpv+xyMiIs5k5leb236v1zsBwDoAcDgcFzPzytpjDofj2/Xr1583duzYBn/sADB+/PjwsrKygtrXRPQugEvt6nSyca5mFFjlIQJMqjQTpqd8CtCiCMP3fF5eXkUw72nOnDnOvftLX9dQ/Q6n8g9z5swZOH/+fK9/Xq2N6wH8NphyA3kNz1gAb/inTZs2s7ePvG9rVuF1iT68BWBUS+oA0IWBus9VsSoH0GTgyczMVJ9s2hr4e1B3McIOnsKs6n7XiPhfAG5uqjyDzUka6um6+omfBfAbv/Of06yG2L2JMBNImJ7yKRE9tmpZThYAtjuns1Ed3YC28PHHHzvXr1//uNb6YwAzAEQ2kbUngFt8Pt8369evv7aV1ToAjFdKrS8oKPhTK8sKxonM/BIzd9RVvQHgAoD/WWka26ZclRLUFe+P+0sTGegXkHzCnv2lU0LfxPp88N0EILxeImHk5BmpQ9u67qPAecy8KGF68vyObkhH6HSBcMOGDY7i4uJXmPlWBH/r34WZn1+3bt29IWgCAchcv3791BCUZefK9evX/64d6rHTX2u8NWlG2hVB5G30Co9At4S4TfVMnDgxHMRzGjumwU1edR176LogP8dOpdMFQq/X+zCAwA/yBwB/IKKzmfkEZr6Ame8DUG/TcSLKLCgomGFTxZPjxo2jcePG0dixYxUzn0BECQA+98/EzPOaKoCZN3q93gi7n3HjxlneotZ4sKCgYEQQ+VrqPV2lutf+sGH2Y4VRRHgGgMcvn5OYc6+ckTaoqYISpicPA3CRX5LfLRhfNmla8tn++SOUeS8bZj//H4Iejfq2B+Zhw+xnlpW865/JiIhNBtC3sbqJkT4hMbG7zf/DUU8Z5pD85blU++OrKIkgpokgFPvnI9aTOqqNHaVTPSMsKCg4HQ2vONYDuGrcuHH+QW8PgI/feOON/2itXwdQ+8CZADy6cePGNcOHD7d95kVEXFPWnrVr136olPoKQI+aw7949dVX4yZNmlTc2HlXXHFFVfPeXZMczJyzYcOGc2s7VkKK4V29Otv/PRSj+ovlrSlXpSzSGmsAdK3JHEOM+wGkNl6UupUOxx8G0wOgw18YRHQLgOtqX+fl5ZUg4MsqYUZqN3C9R1i+V/PyfrB7GzVl1yoB4SVw3e9KpFOrawH83a6czuS1116rAvB6woyU+QD+7/ARdnZUmzpKp7oiJKKbUD+47+rSpcv0gCBYZ8yYMbtN00wAUOmXfGJZWdm05tY9YcKEnwG8598cp9N5QnPLaQki6uf1ehdmZma26+e58uXcjSC6oV5bwIkJCak9A/NOnpxyPIEPX20zCiIcvvsB+PdWpk2bNqtH4LmtNXl60mUAzj+cwguhzEcAmHXtZropMTHRCHXdRwMGAq6Gle3ogs6mU10RMnOC/2siemDEiBEuq3Muv/zyHQUFBQvg19sGYDKA7ObWT0TEflcrhmF4msgaXlBQcKJVWZWVlfubGo5TYzWAMQAiauqeOHz48DsBPNC8VrdO/rKc3ITpKX8BMLgmyYCTxyPg/087cDMBdVcaTHgmLy+vImF68kKA5tYkd/HBcx2Ah0PZRiZ1q/9NuGbMX52X923CjJR1YEysSR5QqR0JAF4JZd1HEs2q35Uz0uq+9BWb3QC6HIxf1WVi2ubqGv5ShzSwA3WaQLhx48YuZWVl9YKLYRirm8rvj5lfIyK/YQd0mlX+xqxfv74HM/v3nHpN02zqlu18ALutyouIiEgGsLSp40T0BYBVzPysX9p969at+9/48ePfaOq8NsAA3sThQAgQD/bPkJiY2KXSxK/9kn48vmds9TASUs/Cv2OLcOPIkSMfeeutt3yhaNwV09MGgPXkwyn09uoVuV8CADQ/C6KJdYeYb0HrAmGXhOkpjQ7DAoBPNjU5JLB9aFqr4D9stl5fohfAcgccv31rwYJKHGM6za2xy+UKvB3TI0eO/CnI0/f4v2DmXsHWu3bt2qiCgoKRzLwah58PAsC7EyZMKAu2nJYYO3bsfAAL/ZIUEWW9/vrrxzV1Tptg2lv/Nep9FpWmIx04nEbA/NoxgzUDht/2y35idPe+9a7sW8MgfTP8v/CrO3kAABEO/SqAb/2yj54yJf2sVlRHAE6w+TlSVRGoyqd8nb7TqDGdJhBGRUUFznJQa9eujQvmXKVUvT9cZm7QweFnzvr16w+uX7/+YEFBwSGllBvABgT0hjLz/UE1vJW01jcC+MovqY9hGFlVVVXt9ryL6zpLahNUwP8f+3dU+Lwwnq93mOjZei9DNJQmISEhEoxr/JL2+8oPLa99kZeXZxLwYr2WOswbQ1H3EYlRCkJx3U/1VWCtaAbPAvMHNb37x5ROc2s8YsQIV0FBwT4AdVdzSqkxAPKCOH2k/wsi2mmRN5yZwy2OM4C7rW5PiegLrfV1TR0HAI/HY9WGOhMmTCh78803k0zT/BCHB46PMk3znmDODwUirveHw9BFtf9OmJY6GmD/AcufKM19rpya1qc2QZHerRnlqGs/j5oyJf2slSuzNreqXWGRVzP7dQQw3lThcWdeOTXtcBKbn4HI7zVmTp06+64WTh/0MjU9bEqxIgY3+fxTa5T5NQXM1NREgNoCI+E/np7I6pkylMO8eGVeXt2XZmZmpvroi69PNYjuZ6C2g7ArQP8G8AscQzNMOk0gBAAiWs3Ms/1e37lhw4YVo0aNavJ50/r163sACBxo2+zpcDXeI6I/jx071nL8HzO7g5n3G6zRo0dvWbdu3U1EVHd1w8x3hKp8K1NmpFykuf7VsDLp8Hxb0nMDnkVdpJSu9951I7O9tWHeDOD6VjSNmAOuLAlJinRSQLbA86K0qroWwCMtqNP76rIlTQ7BqZli12QgJMbe+s1hyznfpOlc//wM/jH4pgI18+y/TkxMTKk0jQMAomsOnXvljLSBq5dl72pOeUezTnNrDABE9Bzqf4udZ5rm00uXLm30NnHt2rVRzLyEmf1voQ9WVVW9bFHNf4noer+fDGaeoLXuM27cuBF2QbCtjB8/fgERLfBLavOpd5Mnp/bRmgJ6GPm1Vaty9wBAQmLiQIBaOkthZmuG0kyeljoBwOktOZfBt3TEUBrtNT4A4P+lffbk6SmTG8s7dWrySSBc5Z9mEL3XWN4gmPAbSgQADviaHBjfGXWqK8IxY8ZsXLdu3RIiSqlNY+br4uLiTlu7du2fSkpK/puUlGSuWbMmPCwsbGLNc7wzAor5U2ODoP1sqemkOOIYhnGDaZrnMvPZ9rlbLjExsWuFT01n4vtQvwOgSvPhgbnsc9xKxC0NKF188PwKwD9acjIT39rCegFgQIV2XAEgvxVlNNvq1dnFCdOTXwWobnomA9mTZyT/oTIybOG6RYvKEhMTjUqvGm8q+jdqhk7V2B5G5v+aW2diYmLXKm08hIDnvD4d0AHWyXWqQAgAkZGRcyorK08PCAYjlFJvxMXFVaxbt+4AEfVq4jnforFjx/6rHZp5fkFBge1tBzM/P378+KDHBY4aNapy/fr1SQA+BhDTmgbWIYxImJFyeNAzQ1Wa6EoNrzeZiK5bvbx6aMrkyZNjmA4/pgBQ5TPM/q/l5e1rqqqEGalJYF7iV/fNiYmJj+Xl5ZlNndOYSTPSTgbr8X5J30UY5kCrciZPT/kjA3+pq7p6KE27BkIAUIaep01jHA6vYhPFTP8OL/M+kTA95cdKEz2hGiwiwiC6w+7/SZvG0skzUuqGxjCja6WJ/ghciILpq9Urchssi9aZdapbY6C60wTVA40bWxeuCxH1Q+AHX21+XFzctTXT5tpaOICBQfw0+9Zw7Nix3xBRo4sLtJADjLi6n8AeYgBglBJz8qplOYvrkozIX/nnJdDLVkEQAI7rEbMCgP9zrgGVPtXsea/E5lz4/W4T8JxdkPAZ5nOo34s67sppyWc2t+7WWpmX9xUTpQIInOLpBHAiGq6kxADuyV+WE8z4xyHM+EXtD6rHfgb+LZSA9GwcQx0lQCcMhAAwduzYA8XFxVcS0WwAO2yyvwNg1Lhx464///zzvTZ5jwpjx47NBfBcO1T1MwgPO+A8edWKJf698wRQvWEopuJnYWP+/PleBhbUT23eUJqpU2d3A2iWX5JPG+YLduetycvbC6p/BagINzSVvy29uiwnXzNfCK5ei9LCl0xqUv7y3FDMJvIB/IrJfGH+8iUfhaC8o0qnuzWulZSUZAJ4iZkXrl27dqjD4fil1noAEUUT0X6tdaFhGAVjxoyxnOGhlNpomuadfkmfNbctRLSLmf/Q3PMA1P1Caq2/Ukrd6ff6fasTHQ7Hb30+3zfM1c/olFJNznjwp7XaYbC+s6njpLiMoX4mYOuqZTmb0ciVw7RpM3v54H2x7ghR1eqXc94Jpn4Y5r/gM+rmhhMRT5w4MbxmgQA4tHOvD57D/5eEegtNmEZVX2j8tfY1g38MZlEGAFA+I1Mr88PD55LbKn9cXFzVj/tK69pCii2/SDMzM3XCtJSgfg9Wr1jyJYAJkxMTB2ufMYoUTgEjFuBygvqeod/OX77kY1hfuT0WOLjdHxExiA+Z0LvDdMTHja3oLYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEKKTa/OlmjpCWlrayVojOTc3+6/+6YmJs040DHNEbm5Wg42ZUlLSrwKoeqczhUqG3q693pV5eXluAEhJzdhhKD4vKyurtLntSUnLyARzEoDyvT/+MCxU+3F0ZikpKf1BanluTvYFDY6lpi8B09O5uYvfak0diYmJ0Q5H2FRmPgXAftOh1uQtXmw3JTPkRo4c6ejb94SpRDgLwH6fwht5WVlf2Z4oQqZTzjXWTHeC6DdJaWmj/NMNQw9gvyW66iFMZGIHK/0JtN6pGOcZjrBPMzIymrX/R2pqxtLk5JlnHn6d2gfMU3NzsoZIEDxypKRkDDccYV9pYAQUdkOprobJa5NT0+9t77b0Oa7fciZOAWiPBuIMjWXJyckntXc7jmWdbq7xzJkze3t8+gJoXKOIbkf1fiJBUaD3c7KzltW8XJScmvGDz6dvAFDvj2POnDnOErf7CsV0qibaq71VuXl5eZ7k5JmnM+t4MvTwpLS0vuz1vsdEU8CoSkpLG6O03gGgKDk5/QIoupRIlxDzypycnP1A9R+nUrrKJHIC6Gkwv6c1TXQ61XqvVycC3AXQL+fm5hYlp6cPI+ZfQmNPbGx0bu1mSP5SUtKnGQY+8jKfZjBdwIzte/f+8Ip/ME5KSxurNM5hhR/LXdEr8vPnl6empl/t83ny8/LyDgJAWlraOczqlJycxXW76qWkpV1bHhGx9LwTTyz7+ptvEqr/L3jb6aeckp+Zmalnz54dUVXly3A61QqPx0wpKTn4/GuvvVaVmpo6gpmGEdFen8/zSu0Vd2Zmptq6dfsUJpwB4q9Nr+Njw2H9nZGaOnM0oC/SxDt+2rNnxSmnnEIuV9lvcnKynqzNk5aWNoBZnZuTs/iVw+el9mTwMjDNWJK7eGNt+uRrrnk8sqJqQ3Ja2s4l2dlZ1e+bowHs1zASiPkQkV5R+3kBQHp6+lk+ptGkuZxIr8rJyfmpuo6MGcy+95VS5zDT2QB9k5OzeDkC5gYnJMyJJJRdHhsTHeX3Gd7nny8lZeb5gHkpFEpNr3dVXs0qPqmpGVOV4o+zsrK+B+quLK/Lzc16BgCS0zJu1t6q/yhHeAqxb21ubu4e//YC5uu5ubnf1fyf9GFWk1lRpIP4zays1m2TcLTpdFeEPp/+LTGeXbIkax0T+iUnz2zRKsUAQIwdUNRgR7sSV9mDYDqdiL9UrM8zHGG5AMCG2QuELtDop4BBYWFhTmI6AUCkAgYBiEtJSbuHFD9ErPcRUzeGeic1NXVIdcl6tGb8jTR+pzRVAejJhDu9Pv0SERQT9QIZ/0tOy7hZab6BWB0CYWapu2xBE28hw9R4xmAaw4ytAI3ue9wJBSNHjnQAQEpK+osGqzlE6gfFdEpUdNn/EhMTezFwqnI6k2sL0Ux3MPix2bNnRwBARkbGccx0a7jLVb512/bVBDqfmbYopmFbt23PAYCysrJIBt/j9Zq5UBzdv39/nZKS8RBDXQ+orwH0NIywgoSEOZEAaOu27atAmK5AW8F0unL4HrX6bJjwBw09gRlbCfTLvsed8OY333zDDFyRnJ5et4cKM90EcMDSYSoDTCty/YIgAKx64QWXJr6dNN0KAFrTMIbxV4a6l5gLiRDHUBsTZ84cCADJqRm/Mhn/UIzdUOxl0Ku1xxg8C2TM11BjmVHK0HempqY3WKY/P39+OQNbSkvdj6anp/erbXbt8eTU9DtB+u+s1AFm1dVwhL2dnl69056GztBax9fm7dWrlxN0eGFcYr7H4Qh/SYHjnU6nLyUl/RpT43nS7AIQBnJkT77mmpjk5IyzGfQqETyk+VtT8+OpqRl1i8MeCzrVFWFCwpxI5rJk0/ScCwDEeJKVvhX1N28PysyZM6O8Pj2HmBYGHluSk/V7v5drUlLTf0hMTDSWZme/nZKW/h1rtWxJ9uIvACAtLW2FZnX+kuys+YmJs05wOMwMn88zNC8vzwMAycnpX7EyHkT1pvIAyJObk5VYc+7JBAwk6NF1V41p6f3AemROTvZVNe95UVR02feZmZmqZg+K+pi/yMnNrt1QaEVKWnpWn+OPvyo1NfV7AANychaPrs2anJaxz3CG38WmekYp8zkATyckzIlklJ0OYHGFx3MFgOVejSQFLCYj7CoGti/JzvpjTRGrU1LTs5OTM87Wuuo7AP219o1ZsmTJzuTk5JNIOS7NzckagZo/9JSUNIqKKktPTk7fzYBjSU7WzMNtSbshYAe6wDf25ZKcrNo/+hWpqWkvHXfcCSmA/jdpmgPgg5EjRzoYmFTmjsqsdyZhMJgbXUWoMiLisy4VVSf75e6Sm5OVXtvm1NT0nwyfvmfkyJHXE/gu0+s5Oy8vr6Lms9xjmHwLgNtRfcKHS7IX/6X6vaasYDI2Amiwl0yYQ03wmnynqfnDlLT0r9jE35YsyVqXmDi7L7H32ogI51kLavYaTkpL2wSmhwEEtQUCkV6QnZ29Zs6cOU6vWfaniDDn2QsW1G1M9S8ASElLf5BNdX3uksUo1QP9AAAILklEQVSfAkBiYuKbhiNsDTrxZveBOlUgjIxxXw3QdlLOYUlpaWDCz6SRlJiY+Mc8m0VBAYDBf0tJTb+LAcPr07FgfjEnN6vBJuvJaRk3E+tpBNLV5yHO7XY7ELDvQyCHQ5/BwEe1QRAAtPa8Zaiwpw+3gQL3PNnifytGjEImVfde8vPnl6ekppd88smeCAANdjEjUgHLX/F/FdOZmhBHoLfrHTLpLRh66pIlC79JSU13pqSkxANlF4FouUm8zND0ZwDLSXOiMpBiMq4nxujU1PS6NjPQVynzRK3xHUCFS5Ys2VndDucQgPunpqav88sbC+Y1MFSkYv5v/bYYb0GZTQdChXrvi5n+C9CZp5126t3btm1/KC0tLU5rdSmIC/Lz5wf8v9AeDtiEvlZkeflgkHF42a7qfUDqrtB8DvW24dM39OnTZwCAHg5H2KrU1PSa90NOMNftka2g36z9d25u7p6U1HT/pfXrLFq06GcAt48cOfKO4447biKUWpCUlpYCeAyAPlngt+F6t+jo/5a6yl5srJyqmBhHZEVVvTSv17seANxudzxYFfoFQX9DlOKHa98HADCjd2N1dFadJhBmZmaqbdu238zAe4pVYt0BwibDGf4bVD93sUTAn4h4tTsiwrfqhRdcjeVJTJx1Imkz5bTTTrms5gqMUlLT620Qbxi60d54rdVuIvNU/7T/b+9uY+yoyjiA/59z5rI3FpVUsTVUg1HZ1sIHYmIIfFjEBlGJLZJL5850634w25akvCSkgKJt2n4QStiSmKoYy9runbnr0PBSaEp4TUQgYiIJ1IpbXivvJBS3l+7OnXMeP9x93+0WAwGy+f++3XvPzDw5dzI55zmT8xjTtgTAS2MxqJ+SGJOhiZ9UoOJ18t0+C9+qybJ/7HiVMwVywEIPe2jHpMbWLYE3rVgUu1SCiqieY+Cvq9eSgbAafyWO47Ocx2CtVvtPGMavwODOdHxEOKZSqcwHxvfnM8a/4iEH06T2/altV0bRTwAsnxbLLDvtGY9vYWK1QSNnwuvzmzZt8mEU7VIvkYpfBmenxeaspLbQJ6Io2pEkycT9KAWwmyfVOladlFox3i8FcNh7/5o18m5RDF90vN2vvTEnXBhbvXr1F44EQX7Pzp2DI7nbvWEULxOVc63on52ivRVXqzcajcZiKF5uBSvHVM1YudLy0NDZU18EybKsCQCDgye/Ou/kxund3d2laflkj8NFIOs+iRXzT4s58yB87rlDPwbkQD3tm7RNfWdn55eazj/V1dW1bWho9vtSFY00TWYr3IRSqRj0KvMHBgaWViqr3wuC4koFTlmwYEHrDvR4U0V+EIarTiuXg4fyfGzwh5GR1qEwWnWLOvM7tcVCUWwXjOd1PmoCvaxajd8qrDwVOP2eQn5ojN6Q582GDU76RRhG16i6O0VKi0V1C8StAIByuVQfGi4eFsGRJEkGRjroDufNn6B6MwA4l6dWTnoiDOODQSCP5N4vtSoXpmlt2vQvSZKnw2rcDKNVv3RNs7NUKhZ6b7rL5eDKRqNxrwRtG6vVeH1R2LuNcYsFWDPbnqMKXBqG0esuMH+zhX4XqsvL5dK3AUBU/6gijwjwVr1/97NTj812735xZRRd5VX+Eobxjcbokw5YJGquhugbRTPfPt5/+FoYRptFNFG1X4fTHkB+mmXZsZXVeE8QlP4QRdFW52ybiLvcWrlxdPHig2g2/fmfaQ7fXK3GtwL+H6ryVSgugZcVSX/thTCKnwmjeLszsiNw7lTnsV1EWjWrRR8FsLla7TwKFPP8LKmEvXtvez+MVmWDg+/fXq1WbxKRsldZD3XXqsWvrdNdURStbzaDt60tLhXRQ2mafuw1Wz4pc2axRNV/Q1Vumvp9a9qhO44da56lKu8A+viMx4v+3Rg9/k7GovvzPG8mSfIuVNY4jw225K5XlX0Q3NZoNDwAGKNbVLAERi8EUPbeHzHQx0ZP44q8E14PGuN/ZSCXQaU7TfseAAA1+KeqDIy2NcYcheDRSXFCD0wtQK/AvqI4POOoRGB+7iFftk63KrBI4M6r1Wr/zbLMuSJf1rpOaauIv8D74kdpmv4LAHp7e49AcIcX/c342XwC0X83GvPuAoAsy466Iu8QQXvhtcdAOqyVnpHGuU4YiQLA5z477xJ4/54N3DYPExvjf9vb2zuUZVlebgs6AHzRWtcjoh0Guk4gD874X6g+aUTXAFhgnW4F5HRXlM4bnfalafoOFM8LZMYpJAD0J0ndWbkAgtOc4nooLoZiWz2phRNHeCpSV4OnAbsRRi9WRTS6yNKf1q7zgodUsVFE13uDu0cfggL81apO2vFZodMeLGnat0e9rPDAIlVzBYDvQN3y/pF83eIzvtklKs9Ypzd4mKp6rEvTvv2t3864XUV/D+haQM4X9VcosO9416snfRtU/f2A3eBVLhdIrV6vv9Zfq90LlWs85GdBUGwRMYPt7e33Ha/v5qI5+UI1tYRhvEfE96Rp+tiJW88NXV1dp+R5Pt+r3FduK509Mb/2/wrDeC2MLKwnfZs+ugjp02jOTI1pBgYNLyfOU80VlUrl80PDzcehUlbF2g/zEAQAFRmG6oc6BxERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERERfbz+BxL4U4Izn2y+AAAAAElFTkSuQmCC @@ -262,6 +242,7 @@ spec: verbs: - create - delete + - get - list - patch - update @@ -294,16 +275,12 @@ spec: resources: - apiservices verbs: + - create + - delete - get - list - - watch - - apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list + - patch + - update - watch - apiGroups: - apps @@ -329,30 +306,6 @@ spec: - statefulsets verbs: - '*' - - apiGroups: - - apps.openshift.io - resources: - - deploymentconfigs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - apps.openshift.io - resources: - - deploymentconfigs/instantiate - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - argoproj.io resources: @@ -688,16 +641,25 @@ spec: resources: - secrets verbs: - - '*' - create + - delete - get + - list + - patch + - update - watch - apiGroups: - "" resources: - secrets/finalizers verbs: - - '*' + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -897,6 +859,15 @@ spec: - patch - update - watch + - apiGroups: + - features.opendatahub.io + resources: + - featuretrackers/status + verbs: + - delete + - get + - patch + - update - apiGroups: - image.openshift.io resources: @@ -1028,6 +999,32 @@ spec: - get - list - patch + - apiGroups: + - modelregistry.opendatahub.io + resources: + - modelregistries + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - modelregistry.opendatahub.io + resources: + - modelregistries/finalizers + verbs: + - update + - apiGroups: + - modelregistry.opendatahub.io + resources: + - modelregistries/status + verbs: + - get + - patch + - update - apiGroups: - monitoring.coreos.com resources: @@ -1234,10 +1231,13 @@ spec: resources: - oauthclients verbs: - - '*' - create - delete - get + - list + - patch + - update + - watch - apiGroups: - opendatahub.io resources: @@ -1615,7 +1615,7 @@ spec: verbs: - '*' - apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io + - trustyai.opendatahub.io resources: - trustyaiservices verbs: @@ -1627,13 +1627,13 @@ spec: - update - watch - apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io + - trustyai.opendatahub.io resources: - trustyaiservices/finalizers verbs: - update - apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io + - trustyai.opendatahub.io resources: - trustyaiservices/status verbs: @@ -1749,6 +1749,9 @@ spec: - --leader-elect command: - /manager + env: + - name: DISABLE_DSC_CONFIG + value: "true" image: REPLACE_IMAGE:latest imagePullPolicy: Always livenessProbe: diff --git a/components/README.md b/components/README.md index 47f812a45f9..962fb3d324d 100644 --- a/components/README.md +++ b/components/README.md @@ -36,6 +36,8 @@ can be found [here](https://github.com/opendatahub-io/opendatahub-operator/tree/ GetComponentName() string GetManagementState() operatorv1.ManagementState SetImageParamsMap(imageMap map[string]string) map[string]string + UpdatePrometheusConfig(cli client.Client, enable bool, component string) error + WaitForDeploymentAvailable(ctx context.Context, r *rest.Config, c string, n string, i int, t int) error } ``` ### Add reconcile and Events @@ -59,3 +61,5 @@ can be found [here](https://github.com/opendatahub-io/opendatahub-operator/tree/ - [ModelMesh Serving](https://github.com/opendatahub-io/opendatahub-operator/tree/main/components/modelmeshserving) - [Workbenches](https://github.com/opendatahub-io/opendatahub-operator/tree/main/components/workbenches) - [TrustyAI](https://github.com/opendatahub-io/opendatahub-operator/tree/main/components/trustyai) +- [ModelRegistry](https://github.com/opendatahub-io/opendatahub-operator/tree/main/components/modelregistry) +- [Kueue](https://github.com/opendatahub-io/kueue) diff --git a/components/codeflare/codeflare.go b/components/codeflare/codeflare.go index 6c1e685a0ec..9aa2338d9a5 100644 --- a/components/codeflare/codeflare.go +++ b/components/codeflare/codeflare.go @@ -3,23 +3,27 @@ package codeflare import ( + "context" "fmt" "path/filepath" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( ComponentName = "codeflare" - CodeflarePath = deploy.DefaultManifestPath + "/" + ComponentName + "/manifests" + CodeflarePath = deploy.DefaultManifestPath + "/" + ComponentName + "/default" CodeflareOperator = "codeflare-operator" RHCodeflareOperator = "rhods-codeflare-operator" + ParamsPath = deploy.DefaultManifestPath + "/" + ComponentName + "/manager" ) // Verifies that CodeFlare implements ComponentInterface. @@ -39,7 +43,7 @@ func (c *CodeFlare) OverrideManifests(_ string) error { return err } // If overlay is defined, update paths - defaultKustomizePath := "manifests" + defaultKustomizePath := "default" if manifestConfig.SourcePath != "" { defaultKustomizePath = manifestConfig.SourcePath } @@ -53,10 +57,10 @@ func (c *CodeFlare) GetComponentName() string { return ComponentName } -func (c *CodeFlare) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { +func (c *CodeFlare) ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { var imageParamMap = map[string]string{ - "odh-codeflare-operator-controller-image": "RELATED_IMAGE_ODH_CODEFLARE_OPERATOR_IMAGE", // no need mcad, embedded in cfo - "namespace": dscispec.ApplicationsNamespace, + "codeflare-operator-controller-image": "RELATED_IMAGE_ODH_CODEFLARE_OPERATOR_IMAGE", // no need mcad, embedded in cfo + "namespace": dscispec.ApplicationsNamespace, } enabled := c.GetManagementState() == operatorv1.Managed monitoringEnabled := dscispec.Monitoring.ManagementState == operatorv1.Managed @@ -65,11 +69,12 @@ func (c *CodeFlare) ReconcileComponent(cli client.Client, owner metav1.Object, d return err } if enabled { - // Download manifests and update paths - if err = c.OverrideManifests(string(platform)); err != nil { - return err + if c.DevFlags != nil { + // Download manifests and update paths + if err = c.OverrideManifests(string(platform)); err != nil { + return err + } } - // check if the CodeFlare operator is installed: it should not be installed dependentOperator := CodeflareOperator // overwrite dependent operator if downstream not match upstream @@ -78,22 +83,22 @@ func (c *CodeFlare) ReconcileComponent(cli client.Client, owner metav1.Object, d } if found, err := deploy.OperatorExists(cli, dependentOperator); err != nil { - return err + return fmt.Errorf("operator exists throws error %w", err) } else if found { return fmt.Errorf("operator %s is found. Please uninstall the operator before enabling %s component", dependentOperator, ComponentName) } // Update image parameters only when we do not have customized manifests set - if dscispec.DevFlags.ManifestsUri == "" && len(c.DevFlags.Manifests) == 0 { - if err := deploy.ApplyParams(CodeflarePath+"/bases", c.SetImageParamsMap(imageParamMap), true); err != nil { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (c.DevFlags == nil || len(c.DevFlags.Manifests) == 0) { + if err := deploy.ApplyParams(ParamsPath, c.SetImageParamsMap(imageParamMap), true); err != nil { return err } } } // Deploy Codeflare - if err := deploy.DeployManifestsFromPath(cli, owner, + if err := deploy.DeployManifestsFromPath(cli, owner, //nolint:revive,nolintlint CodeflarePath, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { @@ -102,6 +107,14 @@ func (c *CodeFlare) ReconcileComponent(cli client.Client, owner metav1.Object, d // CloudServiceMonitoring handling if platform == deploy.ManagedRhods { + if enabled { + // first check if the service is up, so prometheus wont fire alerts when it is just startup + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 20, 2); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentName) + } + // inject prometheus codeflare*.rules in to /opt/manifests/monitoring/prometheus/prometheus-configs.yaml if err = c.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { return err @@ -109,7 +122,7 @@ func (c *CodeFlare) ReconcileComponent(cli client.Client, owner metav1.Object, d if err = deploy.DeployManifestsFromPath(cli, owner, filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), dscispec.Monitoring.Namespace, - ComponentName+"prometheus", true); err != nil { + "prometheus", true); err != nil { return err } } diff --git a/components/component.go b/components/component.go index 56a1b9c0aa7..ccfbb5a23f5 100644 --- a/components/component.go +++ b/components/component.go @@ -1,6 +1,7 @@ package components import ( + "context" "fmt" "os" "path/filepath" @@ -9,6 +10,7 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" "gopkg.in/yaml.v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" @@ -32,7 +34,7 @@ type Component struct { // Add developer fields // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,order=2 - DevFlags DevFlags `json:"devFlags,omitempty"` + DevFlags *DevFlags `json:"devFlags,omitempty"` } func (c *Component) GetManagementState() operatorv1.ManagementState { @@ -78,7 +80,7 @@ type ManifestsConfig struct { } type ComponentInterface interface { - ReconcileComponent(cli client.Client, owner metav1.Object, DSCISpec *dsciv1.DSCInitializationSpec, currentComponentStatus bool) error + ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, DSCISpec *dsciv1.DSCInitializationSpec, currentComponentStatus bool) error Cleanup(cli client.Client, DSCISpec *dsciv1.DSCInitializationSpec) error GetComponentName() string GetManagementState() operatorv1.ManagementState @@ -106,7 +108,7 @@ func (c *Component) UpdatePrometheusConfig(_ client.Client, enable bool, compone DeadManSnitchRules string `yaml:"deadmanssnitch-alerting.rules"` CFRRules string `yaml:"codeflare-recording.rules"` CRARules string `yaml:"codeflare-alerting.rules"` - DashboardRRules string `yaml:"rhods-dashboard-recording.rule"` + DashboardRRules string `yaml:"rhods-dashboard-recording.rules"` DashboardARules string `yaml:"rhods-dashboard-alerting.rules"` DSPRRules string `yaml:"data-science-pipelines-operator-recording.rules"` DSPARules string `yaml:"data-science-pipelines-operator-alerting.rules"` @@ -117,6 +119,12 @@ func (c *Component) UpdatePrometheusConfig(_ client.Client, enable bool, compone RayARules string `yaml:"ray-alerting.rules"` WorkbenchesRRules string `yaml:"workbenches-recording.rules"` WorkbenchesARules string `yaml:"workbenches-alerting.rules"` + KserveRRules string `yaml:"kserve-recording.rules"` + KserveARules string `yaml:"kserve-alerting.rules"` + TrustyAIRRules string `yaml:"trustyai-recording.rules"` + TrustyAIARules string `yaml:"trustyai-alerting.rules"` + KueueRRules string `yaml:"kueue-recording.rules"` + KueueARules string `yaml:"kueue-alerting.rules"` } `yaml:"data"` } var configMap ConfigMap diff --git a/components/dashboard/dashboard.go b/components/dashboard/dashboard.go index 7d83d3eee06..984241832d9 100644 --- a/components/dashboard/dashboard.go +++ b/components/dashboard/dashboard.go @@ -13,6 +13,7 @@ import ( v1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" @@ -20,6 +21,7 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/common" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( @@ -78,7 +80,13 @@ func (d *Dashboard) GetComponentName() string { } //nolint:gocyclo -func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, currentComponentStatus bool) error { +func (d *Dashboard) ReconcileComponent(ctx context.Context, + cli client.Client, + resConf *rest.Config, + owner metav1.Object, + dscispec *dsciv1.DSCInitializationSpec, + currentComponentExist bool, +) error { var imageParamMap = map[string]string{ "odh-dashboard-image": "RELATED_IMAGE_ODH_DASHBOARD_IMAGE", } @@ -92,14 +100,16 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d // Update Default rolebinding if enabled { - if err := d.cleanOauthClientSecrets(cli, dscispec, currentComponentStatus); err != nil { + // cleanup OAuth client related secret and CR if dashboard is in 'installed falas' status + if err := d.cleanOauthClient(cli, dscispec, currentComponentExist); err != nil { return err } - // Download manifests and update paths - if err := d.OverrideManifests(string(platform)); err != nil { - return err + if d.DevFlags != nil { + // Download manifests and update paths + if err := d.OverrideManifests(string(platform)); err != nil { + return err + } } - if platform == deploy.OpenDataHub || platform == "" { err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, "odh-dashboard") if err != nil { @@ -107,7 +117,7 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d } // Deploy CRDs if err := d.deployCRDsForPlatform(cli, owner, dscispec.ApplicationsNamespace, platform); err != nil { - return fmt.Errorf("failed to deploy %s crds %s: %v", ComponentName, PathCRDs, err) + return fmt.Errorf("failed to deploy %s crds %s: %w", ComponentName, PathCRDs, err) } } @@ -118,7 +128,7 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d } // Deploy CRDs if err := d.deployCRDsForPlatform(cli, owner, dscispec.ApplicationsNamespace, platform); err != nil { - return fmt.Errorf("failed to deploy %s crds %s: %v", ComponentNameSupported, PathCRDs, err) + return fmt.Errorf("failed to deploy %s crds %s: %w", ComponentNameSupported, PathCRDs, err) } // Apply RHODS specific configs if err := d.applyRhodsSpecificConfigs(cli, owner, dscispec.ApplicationsNamespace, platform); err != nil { @@ -127,7 +137,7 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d } // Update image parameters (ODH does not use this solution, only downstream) - if dscispec.DevFlags.ManifestsUri == "" && len(d.DevFlags.Manifests) == 0 { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (d.DevFlags == nil || len(d.DevFlags.Manifests) == 0) { if err := deploy.ApplyParams(PathSupported, d.SetImageParamsMap(imageParamMap), false); err != nil { return err } @@ -162,17 +172,24 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d } // CloudService Monitoring handling if platform == deploy.ManagedRhods { + if enabled { + // first check if the service is up, so prometheus wont fire alerts when it is just startup + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentNameSupported, dscispec.ApplicationsNamespace, 20, 3); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentNameSupported) + } + if err := d.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentNameSupported); err != nil { return err } if err = deploy.DeployManifestsFromPath(cli, owner, filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), dscispec.Monitoring.Namespace, - ComponentName+"prometheus", true); err != nil { + "prometheus", true); err != nil { return err } } - return nil default: return nil @@ -263,25 +280,26 @@ func (d *Dashboard) deployConsoleLink(cli client.Client, owner metav1.Object, na return nil } -func (d *Dashboard) cleanOauthClientSecrets(cli client.Client, dscispec *dsciv1.DSCInitializationSpec, currentComponentStatus bool) error { +func (d *Dashboard) cleanOauthClient(cli client.Client, dscispec *dsciv1.DSCInitializationSpec, currentComponentExist bool) error { // Remove previous oauth-client secrets // Check if component is going from state of `Not Installed --> Installed` // Assumption: Component is currently set to enabled - if !currentComponentStatus { + name := "dashboard-oauth-client" + if !currentComponentExist { + fmt.Println("Cleanup any left secret") // Delete client secrets from previous installation oauthClientSecret := &v1.Secret{} err := cli.Get(context.TODO(), client.ObjectKey{ Namespace: dscispec.ApplicationsNamespace, - Name: "dashboard-oauth-client", + Name: name, }, oauthClientSecret) if err != nil { if !apierrs.IsNotFound(err) { - return err + return fmt.Errorf("error getting secret %s: %w", name, err) } } else { - err := cli.Delete(context.TODO(), oauthClientSecret) - if err != nil { - return fmt.Errorf("error deleting oauth client secret: %v", err) + if err := cli.Delete(context.TODO(), oauthClientSecret); err != nil { + return fmt.Errorf("error deleting oauth client secret: %w", err) } } } diff --git a/components/datasciencepipelines/datasciencepipelines.go b/components/datasciencepipelines/datasciencepipelines.go index 0a331974918..5ee08519680 100644 --- a/components/datasciencepipelines/datasciencepipelines.go +++ b/components/datasciencepipelines/datasciencepipelines.go @@ -3,15 +3,19 @@ package datasciencepipelines import ( + "context" + "fmt" "path/filepath" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( @@ -50,7 +54,13 @@ func (d *DataSciencePipelines) GetComponentName() string { return ComponentName } -func (d *DataSciencePipelines) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { +func (d *DataSciencePipelines) ReconcileComponent(ctx context.Context, + cli client.Client, + resConf *rest.Config, + owner metav1.Object, + dscispec *dsciv1.DSCInitializationSpec, + _ bool, +) error { var imageParamMap = map[string]string{ "IMAGES_APISERVER": "RELATED_IMAGE_ODH_ML_PIPELINES_API_SERVER_IMAGE", "IMAGES_ARTIFACT": "RELATED_IMAGE_ODH_ML_PIPELINES_ARTIFACT_MANAGER_IMAGE", @@ -68,15 +78,15 @@ func (d *DataSciencePipelines) ReconcileComponent(cli client.Client, owner metav return err } if enabled { - // Download manifests and update paths - if err = d.OverrideManifests(string(platform)); err != nil { - return err + if d.DevFlags != nil { + // Download manifests and update paths + if err = d.OverrideManifests(string(platform)); err != nil { + return err + } } - // skip check if the dependent operator has beeninstalled, this is done in dashboard - // Update image parameters only when we do not have customized manifests set - if dscispec.DevFlags.ManifestsUri == "" && len(d.DevFlags.Manifests) == 0 { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (d.DevFlags == nil || len(d.DevFlags.Manifests) == 0) { if err := deploy.ApplyParams(Path, d.SetImageParamsMap(imageParamMap), false); err != nil { return err } @@ -88,13 +98,22 @@ func (d *DataSciencePipelines) ReconcileComponent(cli client.Client, owner metav } // CloudService Monitoring handling if platform == deploy.ManagedRhods { + if enabled { + // first check if the service is up, so prometheus wont fire alerts when it is just startup + // only 1 replica should be very quick + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 10, 1); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentName) + } + if err := d.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { return err } if err = deploy.DeployManifestsFromPath(cli, owner, filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), dscispec.Monitoring.Namespace, - ComponentName+"prometheus", true); err != nil { + "prometheus", true); err != nil { return err } } diff --git a/components/kserve/kserve.go b/components/kserve/kserve.go index fd9fd032cf9..e70fe38019a 100644 --- a/components/kserve/kserve.go +++ b/components/kserve/kserve.go @@ -2,6 +2,7 @@ package kserve import ( + "context" "fmt" "path/filepath" "strings" @@ -9,6 +10,7 @@ import ( "github.com/hashicorp/go-multierror" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" @@ -17,6 +19,7 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( @@ -42,34 +45,32 @@ type Kserve struct { func (k *Kserve) OverrideManifests(_ string) error { // Download manifests if defined by devflags - if len(k.DevFlags.Manifests) != 0 { - // Go through each manifests and set the overlays if defined - for _, subcomponent := range k.DevFlags.Manifests { - if strings.Contains(subcomponent.URI, DependentComponentName) { - // Download subcomponent - if err := deploy.DownloadManifests(DependentComponentName, subcomponent); err != nil { - return err - } - // If overlay is defined, update paths - defaultKustomizePath := "base" - if subcomponent.SourcePath != "" { - defaultKustomizePath = subcomponent.SourcePath - } - DependentPath = filepath.Join(deploy.DefaultManifestPath, DependentComponentName, defaultKustomizePath) + // Go through each manifests and set the overlays if defined + for _, subcomponent := range k.DevFlags.Manifests { + if strings.Contains(subcomponent.URI, DependentComponentName) { + // Download subcomponent + if err := deploy.DownloadManifests(DependentComponentName, subcomponent); err != nil { + return err + } + // If overlay is defined, update paths + defaultKustomizePath := "base" + if subcomponent.SourcePath != "" { + defaultKustomizePath = subcomponent.SourcePath } + DependentPath = filepath.Join(deploy.DefaultManifestPath, DependentComponentName, defaultKustomizePath) + } - if strings.Contains(subcomponent.URI, ComponentName) { - // Download subcomponent - if err := deploy.DownloadManifests(ComponentName, subcomponent); err != nil { - return err - } - // If overlay is defined, update paths - defaultKustomizePath := "overlays/odh" - if subcomponent.SourcePath != "" { - defaultKustomizePath = subcomponent.SourcePath - } - Path = filepath.Join(deploy.DefaultManifestPath, ComponentName, defaultKustomizePath) + if strings.Contains(subcomponent.URI, ComponentName) { + // Download subcomponent + if err := deploy.DownloadManifests(ComponentName, subcomponent); err != nil { + return err } + // If overlay is defined, update paths + defaultKustomizePath := "overlays/odh" + if subcomponent.SourcePath != "" { + defaultKustomizePath = subcomponent.SourcePath + } + Path = filepath.Join(deploy.DefaultManifestPath, ComponentName, defaultKustomizePath) } } @@ -80,7 +81,7 @@ func (k *Kserve) GetComponentName() string { return ComponentName } -func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { +func (k *Kserve) ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { // paramMap for Kserve to use. var imageParamMap = map[string]string{} @@ -90,6 +91,7 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci } enabled := k.GetManagementState() == operatorv1.Managed + monitoringEnabled := dscispec.Monitoring.ManagementState == operatorv1.Managed platform, err := deploy.GetPlatform(cli) if err != nil { return err @@ -100,15 +102,16 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci return err } } else { - // Download manifests and update paths - if err = k.OverrideManifests(string(platform)); err != nil { - return err + if k.DevFlags != nil { + // Download manifests and update paths + if err = k.OverrideManifests(string(platform)); err != nil { + return err + } } - // check on dependent operators if all installed in cluster - // dependent operators set in checkRequiredOperatorsInstalled() - if err := checkRequiredOperatorsInstalled(cli); err != nil { - return err + dependOpsErrors := checkDependentOperators(cli).ErrorOrNil() + if dependOpsErrors != nil { + return dependOpsErrors } if err := k.configureServerless(dscispec); err != nil { @@ -116,7 +119,7 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci } // Update image parameters only when we do not have customized manifests set - if dscispec.DevFlags.ManifestsUri == "" && len(k.DevFlags.Manifests) == 0 { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (k.DevFlags == nil || len(k.DevFlags.Manifests) == 0) { if err := deploy.ApplyParams(Path, k.SetImageParamsMap(imageParamMap), false); err != nil { return err } @@ -133,7 +136,7 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci return err } // Update image parameters for odh-model-controller - if dscispec.DevFlags.ManifestsUri == "" && len(k.DevFlags.Manifests) == 0 { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (k.DevFlags == nil || len(k.DevFlags.Manifests) == 0) { if err := deploy.ApplyParams(DependentPath, k.SetImageParamsMap(dependentParamMap), false); err != nil { return err } @@ -141,13 +144,25 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci } if err := deploy.DeployManifestsFromPath(cli, owner, DependentPath, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { - if strings.Contains(err.Error(), "spec.selector") && strings.Contains(err.Error(), "field is immutable") { - // ignore this error - } else { + if !strings.Contains(err.Error(), "spec.selector") || !strings.Contains(err.Error(), "field is immutable") { + // explicitly ignore error if error contains keywords "spec.selector" and "field is immutable" and return all other error. + return err + } + } + // CloudService Monitoring handling + if platform == deploy.ManagedRhods { + if enabled { + // first check if the service is up, so prometheus wont fire alerts when it is just startup + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 20, 2); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoing rules", ComponentName) + } + // kesrve rules + if err := k.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { return err } } - return nil } @@ -156,53 +171,54 @@ func (k *Kserve) Cleanup(_ client.Client, instance *dsciv1.DSCInitializationSpec } func (k *Kserve) configureServerless(instance *dsciv1.DSCInitializationSpec) error { - if k.Serving.ManagementState == operatorv1.Managed { - if instance.ServiceMesh.ManagementState != operatorv1.Managed { - return fmt.Errorf("serviceMesh is not configured as Managaed in DSCI instance but required by KServe serving") - } - - serverlessInitializer := feature.NewFeaturesInitializer(instance, k.configureServerlessFeatures) + switch k.Serving.ManagementState { + case operatorv1.Unmanaged: // Bring your own CR + fmt.Println("Serverless CR is not configured by the operator, we won't do anything") - if err := serverlessInitializer.Prepare(); err != nil { + case operatorv1.Removed: // we remove serving CR + fmt.Println("existing Serverless CR (owned by operator) will be removed") + if err := k.removeServerlessFeatures(instance); err != nil { return err } - if err := serverlessInitializer.Apply(); err != nil { + case operatorv1.Managed: // standard workflow to create CR + switch instance.ServiceMesh.ManagementState { + case operatorv1.Unmanaged, operatorv1.Removed: + return fmt.Errorf("ServiceMesh is need to set to 'Managed' in DSCI CR, it is required by KServe serving field") + } + + serverlessFeatures := feature.ComponentFeaturesHandler(k, instance, k.configureServerlessFeatures()) + + if err := serverlessFeatures.Apply(); err != nil { return err } } - return nil } func (k *Kserve) removeServerlessFeatures(instance *dsciv1.DSCInitializationSpec) error { - serverlessInitializer := feature.NewFeaturesInitializer(instance, k.configureServerlessFeatures) + serverlessFeatures := feature.ComponentFeaturesHandler(k, instance, k.configureServerlessFeatures()) - if err := serverlessInitializer.Prepare(); err != nil { - return err - } - - if err := serverlessInitializer.Delete(); err != nil { - return err - } - return nil + return serverlessFeatures.Delete() } -func checkRequiredOperatorsInstalled(cli client.Client) error { +func checkDependentOperators(cli client.Client) *multierror.Error { var multiErr *multierror.Error - checkAndAppendError := func(operatorName string) { - if found, err := deploy.OperatorExists(cli, operatorName); err != nil { - multiErr = multierror.Append(multiErr, err) - } else if !found { - err = fmt.Errorf("operator %s not found. Please install the operator before enabling %s component", - operatorName, ComponentName) - multiErr = multierror.Append(multiErr, err) - } + if found, err := deploy.OperatorExists(cli, ServiceMeshOperator); err != nil { + multiErr = multierror.Append(multiErr, err) + } else if !found { + err = fmt.Errorf("operator %s not found. Please install the operator before enabling %s component", + ServiceMeshOperator, ComponentName) + multiErr = multierror.Append(multiErr, err) } - checkAndAppendError(ServiceMeshOperator) - checkAndAppendError(ServerlessOperator) - - return multiErr.ErrorOrNil() + if found, err := deploy.OperatorExists(cli, ServerlessOperator); err != nil { + multiErr = multierror.Append(multiErr, err) + } else if !found { + err = fmt.Errorf("operator %s not found. Please install the operator before enabling %s component", + ServerlessOperator, ComponentName) + multiErr = multierror.Append(multiErr, err) + } + return multiErr } diff --git a/components/kserve/serverless_setup.go b/components/kserve/serverless_setup.go index a86f88ad373..95d1d38b882 100644 --- a/components/kserve/serverless_setup.go +++ b/components/kserve/serverless_setup.go @@ -2,68 +2,68 @@ package kserve import ( "path" - "path/filepath" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/serverless" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" ) -const ( - knativeServingNamespace = "knative-serving" - templatesDir = "templates/serverless" -) +func (k *Kserve) configureServerlessFeatures() feature.FeaturesProvider { + return func(handler *feature.FeaturesHandler) error { + servingDeploymentErr := feature.CreateFeature("serverless-serving-deployment"). + For(handler). + Manifests( + path.Join(feature.ServerlessDir, "serving-install"), + ). + WithData(PopulateComponentSettings(k)). + PreConditions( + serverless.EnsureServerlessOperatorInstalled, + serverless.EnsureServerlessAbsent, + servicemesh.EnsureServiceMeshInstalled, + feature.CreateNamespaceIfNotExists(serverless.KnativeServingNamespace), + ). + PostConditions( + feature.WaitForPodsToBeReady(serverless.KnativeServingNamespace), + ). + Load() + if servingDeploymentErr != nil { + return servingDeploymentErr + } -func (k *Kserve) configureServerlessFeatures(s *feature.FeaturesInitializer) error { - var rootDir = filepath.Join(feature.BaseOutputDir, s.DSCInitializationSpec.ApplicationsNamespace) - if err := feature.CopyEmbeddedFiles(templatesDir, rootDir); err != nil { - return err - } + servingNetIstioSecretFilterinErr := feature.CreateFeature("serverless-net-istio-secret-filtering"). + For(handler). + Manifests( + path.Join(feature.ServerlessDir, "serving-net-istio-secret-filtering.patch.tmpl"), + ). + WithData(PopulateComponentSettings(k)). + PreConditions(serverless.EnsureServerlessServingDeployed). + PostConditions( + feature.WaitForPodsToBeReady(serverless.KnativeServingNamespace), + ). + Load() + if servingNetIstioSecretFilterinErr != nil { + return servingNetIstioSecretFilterinErr + } - servingDeployment, err := feature.CreateFeature("serverless-serving-deployment"). - For(s.DSCInitializationSpec). - Manifests( - path.Join(rootDir, templatesDir, "serving-install"), - ). - WithData(PopulateComponentSettings(k)). - PreConditions( - serverless.EnsureServerlessOperatorInstalled, - serverless.EnsureServerlessAbsent, - servicemesh.EnsureServiceMeshInstalled, - feature.CreateNamespaceIfNotExists(knativeServingNamespace), - ). - PostConditions( - feature.WaitForPodsToBeReady(knativeServingNamespace), - ). - Load() - if err != nil { - return err - } - s.Features = append(s.Features, servingDeployment) + serverlessGwErr := feature.CreateFeature("serverless-serving-gateways"). + For(handler). + PreConditions(serverless.EnsureServerlessServingDeployed). + WithData( + PopulateComponentSettings(k), + serverless.ServingDefaultValues, + serverless.ServingIngressDomain, + ). + WithResources(serverless.ServingCertificateResource). + Manifests( + path.Join(feature.ServerlessDir, "serving-istio-gateways"), + ). + Load() + if serverlessGwErr != nil { + return serverlessGwErr + } - servingIstioGateways, err := feature.CreateFeature("serverless-serving-gateways"). - For(s.DSCInitializationSpec). - PreConditions( - // Check serverless is installed - feature.WaitForResourceToBeCreated(knativeServingNamespace, gvr.KnativeServing), - ). - WithData( - serverless.ServingDefaultValues, - serverless.ServingIngressDomain, - PopulateComponentSettings(k), - ). - WithResources(serverless.ServingCertificateResource). - Manifests( - path.Join(rootDir, templatesDir, "serving-istio-gateways"), - ). - Load() - if err != nil { - return err + return nil } - s.Features = append(s.Features, servingIstioGateways) - - return nil } func PopulateComponentSettings(k *Kserve) feature.Action { diff --git a/components/kueue/kueue.go b/components/kueue/kueue.go new file mode 100644 index 00000000000..9dd073957a5 --- /dev/null +++ b/components/kueue/kueue.go @@ -0,0 +1,105 @@ +package kueue + +import ( + "context" + "fmt" + "path/filepath" + + operatorv1 "github.com/openshift/api/operator/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/components" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" +) + +var ( + ComponentName = "kueue" + Path = deploy.DefaultManifestPath + "/" + ComponentName + "/rhoai" // same path for both odh and rhoai +) + +// Verifies that Kueue implements ComponentInterface. +var _ components.ComponentInterface = (*Kueue)(nil) + +// Kueue struct holds the configuration for the Kueue component. +// +kubebuilder:object:generate=true +type Kueue struct { + components.Component `json:""` +} + +func (r *Kueue) OverrideManifests(_ string) error { + // If devflags are set, update default manifests path + if len(r.DevFlags.Manifests) != 0 { + manifestConfig := r.DevFlags.Manifests[0] + if err := deploy.DownloadManifests(ComponentName, manifestConfig); err != nil { + return err + } + // If overlay is defined, update paths + defaultKustomizePath := "openshift" + if manifestConfig.SourcePath != "" { + defaultKustomizePath = manifestConfig.SourcePath + } + Path = filepath.Join(deploy.DefaultManifestPath, ComponentName, defaultKustomizePath) + } + + return nil +} + +func (r *Kueue) GetComponentName() string { + return ComponentName +} + +func (r *Kueue) ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { + var imageParamMap = map[string]string{ + "odh-kueue-controller-image": "RELATED_IMAGE_ODH_KUEUE_OPERATOR_IMAGE", // new kueue image + } + + enabled := r.GetManagementState() == operatorv1.Managed + monitoringEnabled := dscispec.Monitoring.ManagementState == operatorv1.Managed + platform, err := deploy.GetPlatform(cli) + if err != nil { + return err + } + + if enabled { + if r.DevFlags != nil { + // Download manifests and update paths + if err = r.OverrideManifests(string(platform)); err != nil { + return err + } + } + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (r.DevFlags == nil || len(r.DevFlags.Manifests) == 0) { + if err := deploy.ApplyParams(Path, r.SetImageParamsMap(imageParamMap), true); err != nil { + return err + } + } + } + // Deploy Kueue Operator + if err := deploy.DeployManifestsFromPath(cli, owner, Path, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { + return err + } + // CloudService Monitoring handling + if platform == deploy.ManagedRhods { + if enabled { + // first check if the service is up, so prometheus wont fire alerts when it is just startup + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 20, 2); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentName) + } + if err := r.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { + return err + } + if err = deploy.DeployManifestsFromPath(cli, owner, + filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), + dscispec.Monitoring.Namespace, + "prometheus", true); err != nil { + return err + } + } + + return nil +} diff --git a/components/kueue/zz_generated.deepcopy.go b/components/kueue/zz_generated.deepcopy.go new file mode 100644 index 00000000000..2acbd510ffa --- /dev/null +++ b/components/kueue/zz_generated.deepcopy.go @@ -0,0 +1,40 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package kueue + +import () + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Kueue) DeepCopyInto(out *Kueue) { + *out = *in + in.Component.DeepCopyInto(&out.Component) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kueue. +func (in *Kueue) DeepCopy() *Kueue { + if in == nil { + return nil + } + out := new(Kueue) + in.DeepCopyInto(out) + return out +} diff --git a/components/modelmeshserving/modelmeshserving.go b/components/modelmeshserving/modelmeshserving.go index 75acf6d72e3..98ecee30c9f 100644 --- a/components/modelmeshserving/modelmeshserving.go +++ b/components/modelmeshserving/modelmeshserving.go @@ -2,17 +2,21 @@ package modelmeshserving import ( + "context" + "fmt" "path/filepath" "strings" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( @@ -60,7 +64,6 @@ func (m *ModelMeshServing) OverrideManifests(_ string) error { Path = filepath.Join(deploy.DefaultManifestPath, ComponentName, defaultKustomizePath) } } - return nil } @@ -68,7 +71,13 @@ func (m *ModelMeshServing) GetComponentName() string { return ComponentName } -func (m *ModelMeshServing) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { +func (m *ModelMeshServing) ReconcileComponent(ctx context.Context, + cli client.Client, + resConf *rest.Config, + owner metav1.Object, + dscispec *dsciv1.DSCInitializationSpec, + _ bool, +) error { var imageParamMap = map[string]string{ "odh-mm-rest-proxy": "RELATED_IMAGE_ODH_MM_REST_PROXY_IMAGE", "odh-modelmesh-runtime-adapter": "RELATED_IMAGE_ODH_MODELMESH_RUNTIME_ADAPTER_IMAGE", @@ -91,9 +100,11 @@ func (m *ModelMeshServing) ReconcileComponent(cli client.Client, owner metav1.Ob // Update Default rolebinding if enabled { - // Download manifests and update paths - if err = m.OverrideManifests(string(platform)); err != nil { - return err + if m.DevFlags != nil { + // Download manifests and update paths + if err = m.OverrideManifests(string(platform)); err != nil { + return err + } } if err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, @@ -104,7 +115,7 @@ func (m *ModelMeshServing) ReconcileComponent(cli client.Client, owner metav1.Ob return err } // Update image parameters - if dscispec.DevFlags.ManifestsUri == "" && len(m.DevFlags.Manifests) == 0 { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (m.DevFlags == nil || len(m.DevFlags.Manifests) == 0) { if err := deploy.ApplyParams(Path, m.SetImageParamsMap(imageParamMap), false); err != nil { return err } @@ -118,27 +129,33 @@ func (m *ModelMeshServing) ReconcileComponent(cli client.Client, owner metav1.Ob // For odh-model-controller if enabled { - err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, "odh-model-controller") - if err != nil { + if err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, + "odh-model-controller"); err != nil { return err } // Update image parameters for odh-model-controller - if dscispec.DevFlags.ManifestsUri == "" { + if dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "" { if err := deploy.ApplyParams(DependentPath, m.SetImageParamsMap(dependentImageParamMap), false); err != nil { return err } } } if err := deploy.DeployManifestsFromPath(cli, owner, DependentPath, dscispec.ApplicationsNamespace, m.GetComponentName(), enabled); err != nil { - if strings.Contains(err.Error(), "spec.selector") && strings.Contains(err.Error(), "field is immutable") { - // ignore this error - } else { + // explicitly ignore error if error contains keywords "spec.selector" and "field is immutable" and return all other error. + if !strings.Contains(err.Error(), "spec.selector") || !strings.Contains(err.Error(), "field is immutable") { return err } } // CloudService Monitoring handling if platform == deploy.ManagedRhods { + if enabled { + // first check if service is up, so prometheus wont fire alerts when it is just startup + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 20, 2); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentName) + } // first model-mesh rules if err := m.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { return err @@ -150,7 +167,7 @@ func (m *ModelMeshServing) ReconcileComponent(cli client.Client, owner metav1.Ob if err = deploy.DeployManifestsFromPath(cli, owner, filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), dscispec.Monitoring.Namespace, - ComponentName+"prometheus", true); err != nil { + "prometheus", true); err != nil { return err } } diff --git a/components/modelregistry/modelregistry.go b/components/modelregistry/modelregistry.go new file mode 100644 index 00000000000..aaf4c5297be --- /dev/null +++ b/components/modelregistry/modelregistry.go @@ -0,0 +1,87 @@ +// Package modelregistry provides utility functions to config ModelRegistry, an ML Model metadata repository service +package modelregistry + +import ( + "context" + "path/filepath" + + operatorv1 "github.com/openshift/api/operator/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/components" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" +) + +var ( + ComponentName = "model-registry-operator" + Path = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays/odh" +) + +// Verifies that ModelRegistry implements ComponentInterface. +var _ components.ComponentInterface = (*ModelRegistry)(nil) + +// ModelRegistry struct holds the configuration for the ModelRegistry component. +// +kubebuilder:object:generate=true +type ModelRegistry struct { + components.Component `json:""` +} + +func (m *ModelRegistry) OverrideManifests(_ string) error { + // If devflags are set, update default manifests path + if len(m.DevFlags.Manifests) != 0 { + manifestConfig := m.DevFlags.Manifests[0] + if err := deploy.DownloadManifests(ComponentName, manifestConfig); err != nil { + return err + } + // If overlay is defined, update paths + defaultKustomizePath := "overlays/odh" + if manifestConfig.SourcePath != "" { + defaultKustomizePath = manifestConfig.SourcePath + } + Path = filepath.Join(deploy.DefaultManifestPath, ComponentName, defaultKustomizePath) + } + + return nil +} + +func (m *ModelRegistry) GetComponentName() string { + return ComponentName +} + +func (m *ModelRegistry) ReconcileComponent(_ context.Context, cli client.Client, _ *rest.Config, + owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { + var imageParamMap = map[string]string{ + "IMAGES_MODELREGISTRY_OPERATOR": "RELATED_IMAGE_ODH_MODELREGISTRY_OPERATOR_IMAGE", + "IMAGES_GRPC_SERVICE": "RELATED_IMAGE_ODH_MODELREGISTRY_GRPC_SERVICE_IMAGE", + "IMAGES_REST_SERVICE": "RELATED_IMAGE_ODH_MODELREGISTRY_REST_SERVICE_IMAGE", + } + enabled := m.GetManagementState() == operatorv1.Managed + + platform, err := deploy.GetPlatform(cli) + if err != nil { + return err + } + + if enabled { + if m.DevFlags != nil { + // Download manifests and update paths + if err = m.OverrideManifests(string(platform)); err != nil { + return err + } + } + + // Update image parameters only when we do not have customized manifests set + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (m.DevFlags == nil || len(m.DevFlags.Manifests) == 0) { + if err := deploy.ApplyParams(Path, m.SetImageParamsMap(imageParamMap), false); err != nil { + return err + } + } + } + // Deploy ModelRegistry Operator + err = deploy.DeployManifestsFromPath(cli, owner, Path, dscispec.ApplicationsNamespace, m.GetComponentName(), enabled) + + return err +} diff --git a/components/modelregistry/zz_generated.deepcopy.go b/components/modelregistry/zz_generated.deepcopy.go new file mode 100644 index 00000000000..3ed241dd7f1 --- /dev/null +++ b/components/modelregistry/zz_generated.deepcopy.go @@ -0,0 +1,40 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package modelregistry + +import () + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelRegistry) DeepCopyInto(out *ModelRegistry) { + *out = *in + in.Component.DeepCopyInto(&out.Component) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelRegistry. +func (in *ModelRegistry) DeepCopy() *ModelRegistry { + if in == nil { + return nil + } + out := new(ModelRegistry) + in.DeepCopyInto(out) + return out +} diff --git a/components/ray/ray.go b/components/ray/ray.go index 0d0e2a90e0f..f891e3b6ae5 100644 --- a/components/ray/ray.go +++ b/components/ray/ray.go @@ -3,15 +3,19 @@ package ray import ( + "context" + "fmt" "path/filepath" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( @@ -50,7 +54,7 @@ func (r *Ray) GetComponentName() string { return ComponentName } -func (r *Ray) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { +func (r *Ray) ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { var imageParamMap = map[string]string{ "odh-kuberay-operator-controller-image": "RELATED_IMAGE_ODH_KUBERAY_OPERATOR_CONTROLLER_IMAGE", "namespace": dscispec.ApplicationsNamespace, @@ -64,12 +68,13 @@ func (r *Ray) ReconcileComponent(cli client.Client, owner metav1.Object, dscispe } if enabled { - // Download manifests and update paths - if err = r.OverrideManifests(string(platform)); err != nil { - return err + if r.DevFlags != nil { + // Download manifests and update paths + if err = r.OverrideManifests(string(platform)); err != nil { + return err + } } - - if dscispec.DevFlags.ManifestsUri == "" || len(r.DevFlags.Manifests) == 0 { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (r.DevFlags == nil || len(r.DevFlags.Manifests) == 0) { if err := deploy.ApplyParams(RayPath, r.SetImageParamsMap(imageParamMap), true); err != nil { return err } @@ -81,13 +86,20 @@ func (r *Ray) ReconcileComponent(cli client.Client, owner metav1.Object, dscispe } // CloudService Monitoring handling if platform == deploy.ManagedRhods { + if enabled { + // first check if the service is up, so prometheus wont fire alerts when it is just startup + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 20, 2); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentName) + } if err := r.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { return err } if err = deploy.DeployManifestsFromPath(cli, owner, filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), dscispec.Monitoring.Namespace, - ComponentName+"prometheus", true); err != nil { + "prometheus", true); err != nil { return err } } diff --git a/components/trustyai/trustyai.go b/components/trustyai/trustyai.go index ee3da9062a5..5cc182a0e91 100644 --- a/components/trustyai/trustyai.go +++ b/components/trustyai/trustyai.go @@ -2,15 +2,19 @@ package trustyai import ( + "context" + "fmt" "path/filepath" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( @@ -41,7 +45,6 @@ func (t *TrustyAI) OverrideManifests(_ string) error { } Path = filepath.Join(deploy.DefaultManifestPath, ComponentName, defaultKustomizePath) } - return nil } @@ -49,12 +52,13 @@ func (t *TrustyAI) GetComponentName() string { return ComponentName } -func (t *TrustyAI) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { +func (t *TrustyAI) ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, dscispec *dsciv1.DSCInitializationSpec, _ bool) error { var imageParamMap = map[string]string{ "trustyaiServiceImage": "RELATED_IMAGE_ODH_TRUSTYAI_SERVICE_IMAGE", "trustyaiOperatorImage": "RELATED_IMAGE_ODH_TRUSTYAI_SERVICE_OPERATOR_IMAGE", } enabled := t.GetManagementState() == operatorv1.Managed + monitoringEnabled := dscispec.Monitoring.ManagementState == operatorv1.Managed platform, err := deploy.GetPlatform(cli) if err != nil { @@ -62,19 +66,40 @@ func (t *TrustyAI) ReconcileComponent(cli client.Client, owner metav1.Object, ds } if enabled { - // Download manifests and update paths - if err = t.OverrideManifests(string(platform)); err != nil { - return err + if t.DevFlags != nil { + // Download manifests and update paths + if err = t.OverrideManifests(string(platform)); err != nil { + return err + } } - - if dscispec.DevFlags.ManifestsUri == "" { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (t.DevFlags == nil || len(t.DevFlags.Manifests) == 0) { if err := deploy.ApplyParams(Path, t.SetImageParamsMap(imageParamMap), false); err != nil { return err } } } // Deploy TrustyAI Operator - err = deploy.DeployManifestsFromPath(cli, owner, Path, dscispec.ApplicationsNamespace, t.GetComponentName(), enabled) + if err := deploy.DeployManifestsFromPath(cli, owner, Path, dscispec.ApplicationsNamespace, t.GetComponentName(), enabled); err != nil { + return err + } - return err + // CloudService Monitoring handling + if platform == deploy.ManagedRhods { + if enabled { + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 10, 1); err != nil { + return fmt.Errorf("deployment for %s is not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployment for %s is done, updating monitoring rules\n", ComponentName) + } + if err := t.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { + return err + } + if err = deploy.DeployManifestsFromPath(cli, owner, + filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), + dscispec.Monitoring.Namespace, + "prometheus", true); err != nil { + return err + } + } + return nil } diff --git a/components/workbenches/workbenches.go b/components/workbenches/workbenches.go index f3b645d2999..4abd5c74f14 100644 --- a/components/workbenches/workbenches.go +++ b/components/workbenches/workbenches.go @@ -2,17 +2,21 @@ package workbenches import ( + "context" + "fmt" "path/filepath" "strings" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/monitoring" ) var ( @@ -37,56 +41,53 @@ type Workbenches struct { func (w *Workbenches) OverrideManifests(platform string) error { // Download manifests if defined by devflags - if len(w.DevFlags.Manifests) != 0 { - // Go through each manifests and set the overlays if defined - for _, subcomponent := range w.DevFlags.Manifests { - if strings.Contains(subcomponent.URI, DependentComponentName) { - // Download subcomponent - if err := deploy.DownloadManifests(DependentComponentName, subcomponent); err != nil { - return err - } - // If overlay is defined, update paths - defaultKustomizePath := "overlays/additional" - defaultKustomizePathSupported := "notebook-images/overlays/additional" - if subcomponent.SourcePath != "" { - defaultKustomizePath = subcomponent.SourcePath - defaultKustomizePathSupported = subcomponent.SourcePath - } - if platform == string(deploy.ManagedRhods) || platform == string(deploy.SelfManagedRhods) { - notebookImagesPathSupported = filepath.Join(deploy.DefaultManifestPath, "jupyterhub", defaultKustomizePathSupported) - } else { - notebookImagesPath = filepath.Join(deploy.DefaultManifestPath, DependentComponentName, defaultKustomizePath) - } + // Go through each manifests and set the overlays if defined + for _, subcomponent := range w.DevFlags.Manifests { + if strings.Contains(subcomponent.URI, DependentComponentName) { + // Download subcomponent + if err := deploy.DownloadManifests(DependentComponentName, subcomponent); err != nil { + return err + } + // If overlay is defined, update paths + defaultKustomizePath := "overlays/additional" + defaultKustomizePathSupported := "notebook-images/overlays/additional" + if subcomponent.SourcePath != "" { + defaultKustomizePath = subcomponent.SourcePath + defaultKustomizePathSupported = subcomponent.SourcePath + } + if platform == string(deploy.ManagedRhods) || platform == string(deploy.SelfManagedRhods) { + notebookImagesPathSupported = filepath.Join(deploy.DefaultManifestPath, "jupyterhub", defaultKustomizePathSupported) + } else { + notebookImagesPath = filepath.Join(deploy.DefaultManifestPath, DependentComponentName, defaultKustomizePath) } + } - if strings.Contains(subcomponent.ContextDir, "components/odh-notebook-controller") { - // Download subcomponent - if err := deploy.DownloadManifests("odh-notebook-controller/odh-notebook-controller", subcomponent); err != nil { - return err - } - // If overlay is defined, update paths - defaultKustomizePathNbc := "base" - if subcomponent.SourcePath != "" { - defaultKustomizePathNbc = subcomponent.SourcePath - } - notebookControllerPath = filepath.Join(deploy.DefaultManifestPath, "odh-notebook-controller/odh-notebook-controller", defaultKustomizePathNbc) + if strings.Contains(subcomponent.ContextDir, "components/odh-notebook-controller") { + // Download subcomponent + if err := deploy.DownloadManifests("odh-notebook-controller/odh-notebook-controller", subcomponent); err != nil { + return err + } + // If overlay is defined, update paths + defaultKustomizePathNbc := "base" + if subcomponent.SourcePath != "" { + defaultKustomizePathNbc = subcomponent.SourcePath } + notebookControllerPath = filepath.Join(deploy.DefaultManifestPath, "odh-notebook-controller/odh-notebook-controller", defaultKustomizePathNbc) + } - if strings.Contains(subcomponent.ContextDir, "components/notebook-controller") { - // Download subcomponent - if err := deploy.DownloadManifests("odh-notebook-controller/kf-notebook-controller", subcomponent); err != nil { - return err - } - // If overlay is defined, update paths - defaultKustomizePathKfNbc := "overlays/openshift" - if subcomponent.SourcePath != "" { - defaultKustomizePathKfNbc = subcomponent.SourcePath - } - kfnotebookControllerPath = filepath.Join(deploy.DefaultManifestPath, "odh-notebook-controller/kf-notebook-controller", defaultKustomizePathKfNbc) + if strings.Contains(subcomponent.ContextDir, "components/notebook-controller") { + // Download subcomponent + if err := deploy.DownloadManifests("odh-notebook-controller/kf-notebook-controller", subcomponent); err != nil { + return err + } + // If overlay is defined, update paths + defaultKustomizePathKfNbc := "overlays/openshift" + if subcomponent.SourcePath != "" { + defaultKustomizePathKfNbc = subcomponent.SourcePath } + kfnotebookControllerPath = filepath.Join(deploy.DefaultManifestPath, "odh-notebook-controller/kf-notebook-controller", defaultKustomizePathKfNbc) } } - return nil } @@ -94,7 +95,7 @@ func (w *Workbenches) GetComponentName() string { return ComponentName } -func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsci.DSCInitializationSpec, _ bool) error { +func (w *Workbenches) ReconcileComponent(ctx context.Context, cli client.Client, resConf *rest.Config, owner metav1.Object, dscispec *dsci.DSCInitializationSpec, _ bool) error { var imageParamMap = map[string]string{ "odh-notebook-controller-image": "RELATED_IMAGE_ODH_NOTEBOOK_CONTROLLER_IMAGE", "odh-kf-notebook-controller-image": "RELATED_IMAGE_ODH_KF_NOTEBOOK_CONTROLLER_IMAGE", @@ -112,11 +113,12 @@ func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, // Set default notebooks namespace // Create rhods-notebooks namespace in managed platforms if enabled { - // Download manifests and update paths - if err = w.OverrideManifests(string(platform)); err != nil { - return err + if w.DevFlags != nil { + // Download manifests and update paths + if err = w.OverrideManifests(string(platform)); err != nil { + return err + } } - if platform == deploy.SelfManagedRhods || platform == deploy.ManagedRhods { _, err := cluster.CreateNamespace(cli, "rhods-notebooks", cluster.WithLabels(cluster.ODHGeneratedNamespaceLabel, "true")) if err != nil { @@ -137,7 +139,7 @@ func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, // Update image parameters for nbc in downstream if enabled { - if dscispec.DevFlags.ManifestsUri == "" && len(w.DevFlags.Manifests) == 0 { + if (dscispec.DevFlags == nil || dscispec.DevFlags.ManifestsUri == "") && (w.DevFlags == nil || len(w.DevFlags.Manifests) == 0) { if platform == deploy.ManagedRhods || platform == deploy.SelfManagedRhods { // for kf-notebook-controller image if err := deploy.ApplyParams(notebookControllerPath, w.SetImageParamsMap(imageParamMap), false); err != nil { @@ -172,13 +174,21 @@ func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, } // CloudService Monitoring handling if platform == deploy.ManagedRhods { + if enabled { + // first check if the service is up, so prometheus wont fire alerts when it is just startup + // only 1 replica set timeout to 1min + if err := monitoring.WaitForDeploymentAvailable(ctx, resConf, ComponentName, dscispec.ApplicationsNamespace, 10, 1); err != nil { + return fmt.Errorf("deployments for %s are not ready to server: %w", ComponentName, err) + } + fmt.Printf("deployments for %s are done, updating monitoring rules\n", ComponentName) + } if err := w.UpdatePrometheusConfig(cli, enabled && monitoringEnabled, ComponentName); err != nil { return err } if err = deploy.DeployManifestsFromPath(cli, owner, filepath.Join(deploy.DefaultManifestPath, "monitoring", "prometheus", "apps"), dscispec.Monitoring.Namespace, - ComponentName+"prometheus", true); err != nil { + "prometheus", true); err != nil { return err } } diff --git a/components/zz_generated.deepcopy.go b/components/zz_generated.deepcopy.go index e9af55fc108..e7968282dc9 100644 --- a/components/zz_generated.deepcopy.go +++ b/components/zz_generated.deepcopy.go @@ -26,7 +26,11 @@ import () // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Component) DeepCopyInto(out *Component) { *out = *in - in.DevFlags.DeepCopyInto(&out.DevFlags) + if in.DevFlags != nil { + in, out := &in.DevFlags, &out.DevFlags + *out = new(DevFlags) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Component. diff --git a/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml b/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml index deb66af4137..460f11405b7 100644 --- a/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml +++ b/config/crd/bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml @@ -36,7 +36,7 @@ spec: metadata: type: object spec: - description: DataScienceCluster defines the desired state of the cluster. + description: DataScienceClusterSpec defines the desired state of the cluster. properties: components: description: Override and fine tune specific component configurations. @@ -263,9 +263,10 @@ spec: type: string type: object managementState: - default: Removed + default: Managed enum: - Managed + - Unmanaged - Removed pattern: ^(Managed|Unmanaged|Force|Removed)$ type: string @@ -278,6 +279,49 @@ spec: type: string type: object type: object + kueue: + description: Kueue component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: "" + description: contextDir is the relative path to + the folder containing manifests in a repository + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: "Set to one of the following values: \n - \"Managed\" + : the operator is actively managing the component and trying + to keep it active. It will only upgrade the component if + it is safe to do so \n - \"Removed\" : the operator is actively + managing the component and will not install it, or if it + is installed, the operator will try to remove it" + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object modelmeshserving: description: ModelMeshServing component configuration. Does not support enabled Kserve at the same time @@ -322,6 +366,49 @@ spec: pattern: ^(Managed|Unmanaged|Force|Removed)$ type: string type: object + modelregistry: + description: ModelRegistry component configuration. + properties: + devFlags: + description: Add developer fields + properties: + manifests: + description: List of custom manifests for the given component + items: + properties: + contextDir: + default: "" + description: contextDir is the relative path to + the folder containing manifests in a repository + type: string + sourcePath: + default: "" + description: 'sourcePath is the subpath within contextDir + where kustomize builds start. Examples include + any sub-folder or path: `base`, `overlays/dev`, + `default`, `odh` etc' + type: string + uri: + default: "" + description: uri is the URI point to a git repo + with tag/branch. e.g https://github.com/org/repo/tarball/ + type: string + type: object + type: array + type: object + managementState: + description: "Set to one of the following values: \n - \"Managed\" + : the operator is actively managing the component and trying + to keep it active. It will only upgrade the component if + it is safe to do so \n - \"Removed\" : the operator is actively + managing the component and will not install it, or if it + is installed, the operator will try to remove it" + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + type: object ray: description: Ray component configuration. properties: diff --git a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml index e805f082618..6e701ec8895 100644 --- a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml +++ b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -119,6 +119,7 @@ spec: default: Removed enum: - Managed + - Unmanaged - Removed pattern: ^(Managed|Unmanaged|Force|Removed)$ type: string diff --git a/config/crd/bases/features.opendatahub.io_featuretrackers.yaml b/config/crd/bases/features.opendatahub.io_featuretrackers.yaml index dc544d8f79f..c10e24d7c0c 100644 --- a/config/crd/bases/features.opendatahub.io_featuretrackers.yaml +++ b/config/crd/bases/features.opendatahub.io_featuretrackers.yaml @@ -41,10 +41,51 @@ spec: type: object spec: description: FeatureTrackerSpec defines the desired state of FeatureTracker. + properties: + appNamespace: + type: string + source: + description: Source describes the type of object that created the + related Feature to this FeatureTracker. + properties: + name: + type: string + type: + type: string + type: object type: object status: description: FeatureTrackerStatus defines the observed state of FeatureTracker. + properties: + conditions: + items: + description: Condition represents the state of the operator's reconciliation + functionality. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: ConditionType is the state of the operator's reconciliation + functionality. + type: string + required: + - status + - type + type: object + type: array type: object type: object served: true storage: true + subresources: + status: {} diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml.in similarity index 100% rename from config/manager/kustomization.yaml rename to config/manager/kustomization.yaml.in diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 1771ba99719..c2308fe4bde 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -36,6 +36,9 @@ spec: containers: - command: - /manager + env: + - name: DISABLE_DSC_CONFIG + value: 'true' args: - --leader-elect image: controller:latest diff --git a/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml b/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml index bd90811a8d9..93a534c5fd5 100644 --- a/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml @@ -9,49 +9,6 @@ metadata: containerImage: quay.io/opendatahub/opendatahub-operator:v2.1.0 createdAt: "2023-8-23T00:00:00Z" olm.skipRange: '>=1.0.0 <2.0.0' - operatorframework.io/initialization-resource: |- - { - "apiVersion": "datasciencecluster.opendatahub.io/v1", - "kind": "DataScienceCluster", - "metadata": { - "name": "default", - "labels": { - "app.kubernetes.io/name": "datasciencecluster", - "app.kubernetes.io/instance": "default", - "app.kubernetes.io/part-of": "opendatahub-operator", - "app.kubernetes.io/managed-by": "kustomize", - "app.kubernetes.io/created-by": "opendatahub-operator" - } - }, - "spec": { - "components": { - "codeflare": { - "managementState": "Removed" - }, - "dashboard": { - "managementState": "Managed" - }, - "datasciencepipelines": { - "managementState": "Managed" - }, - "kserve": { - "managementState": "Removed" - }, - "modelmeshserving": { - "managementState": "Managed" - }, - "ray": { - "managementState": "Removed" - }, - "workbenches": { - "managementState": "Managed" - }, - "trustyai": { - "managementState": "Removed" - } - } - } - } operators.operatorframework.io/internal-objects: '[dscinitialization.opendatahub.io]' repository: https://github.com/opendatahub-io/opendatahub-operator name: opendatahub-operator.v0.0.0 diff --git a/config/manifests/description-patch.yml b/config/manifests/description-patch.yml index 0b6d1e84da5..74d47766222 100644 --- a/config/manifests/description-patch.yml +++ b/config/manifests/description-patch.yml @@ -15,6 +15,6 @@ spec: * Jupyter Notebooks - JupyterLab notebook that provide Python support for GPU workloads * Data Science Pipelines - Pipeline solution for end to end MLOps workflows that support the Kubeflow Pipelines SDK and Tekton * Model Mesh - ModelMesh Serving is the Controller for managing ModelMesh, a general-purpose model serving management/routing layer - * Distributed Workloads(Incubation) - Stack built to make managing distributed compute infrastructure in the cloud easy and intuitive for Data Scientists. This stack consists of two components - Codeflare and KubeRay. - * Kserve (Incubation) - Kserve is the Controller for for serving machine learning (ML) models on arbitrary frameworks + * Distributed Workloads(Incubation) - Stack built to make managing distributed compute infrastructure in the cloud easy and intuitive for Data Scientists. This stack consists of three components + Codeflare , KubeRay and Kueue. + * Kserve - Kserve is the Controller for for serving machine learning (ML) models on arbitrary frameworks diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index eb4c416e902..1624ddca325 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -56,6 +56,7 @@ rules: verbs: - create - delete + - get - list - patch - update @@ -88,16 +89,12 @@ rules: resources: - apiservices verbs: + - create + - delete - get - list - - watch -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list + - patch + - update - watch - apiGroups: - apps @@ -123,30 +120,6 @@ rules: - statefulsets verbs: - '*' -- apiGroups: - - apps.openshift.io - resources: - - deploymentconfigs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps.openshift.io - resources: - - deploymentconfigs/instantiate - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - argoproj.io resources: @@ -482,16 +455,25 @@ rules: resources: - secrets verbs: - - '*' - create + - delete - get + - list + - patch + - update - watch - apiGroups: - "" resources: - secrets/finalizers verbs: - - '*' + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -691,6 +673,15 @@ rules: - patch - update - watch +- apiGroups: + - features.opendatahub.io + resources: + - featuretrackers/status + verbs: + - delete + - get + - patch + - update - apiGroups: - image.openshift.io resources: @@ -822,6 +813,32 @@ rules: - get - list - patch +- apiGroups: + - modelregistry.opendatahub.io + resources: + - modelregistries + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - modelregistry.opendatahub.io + resources: + - modelregistries/finalizers + verbs: + - update +- apiGroups: + - modelregistry.opendatahub.io + resources: + - modelregistries/status + verbs: + - get + - patch + - update - apiGroups: - monitoring.coreos.com resources: @@ -1028,10 +1045,13 @@ rules: resources: - oauthclients verbs: - - '*' - create - delete - get + - list + - patch + - update + - watch - apiGroups: - opendatahub.io resources: @@ -1409,7 +1429,7 @@ rules: verbs: - '*' - apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io + - trustyai.opendatahub.io resources: - trustyaiservices verbs: @@ -1421,13 +1441,13 @@ rules: - update - watch - apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io + - trustyai.opendatahub.io resources: - trustyaiservices/finalizers verbs: - update - apiGroups: - - trustyai.opendatahub.io.trustyai.opendatahub.io + - trustyai.opendatahub.io resources: - trustyaiservices/status verbs: diff --git a/config/samples/datasciencecluster_v1_datasciencecluster.yaml b/config/samples/datasciencecluster_v1_datasciencecluster.yaml index 2941c437667..6851cbdb3e9 100644 --- a/config/samples/datasciencecluster_v1_datasciencecluster.yaml +++ b/config/samples/datasciencecluster_v1_datasciencecluster.yaml @@ -16,13 +16,27 @@ spec: managementState: "Managed" datasciencepipelines: managementState: "Managed" - kserve: - managementState: "Removed" + kserve: { + managementState: "Managed", + serving: { + ingressGateway: { + certificate: { + type: SelfSigned + } + }, + name: "knative-serving", + managementState: "Managed" + } + } modelmeshserving: managementState: "Managed" + kueue: + managementState: "Removed" ray: managementState: "Removed" workbenches: managementState: "Managed" trustyai: + managementState: "Managed" + modelregistry: managementState: "Removed" diff --git a/config/samples/dscinitialization_v1_dscinitialization.yaml b/config/samples/dscinitialization_v1_dscinitialization.yaml index 83e04c2749e..b367698be05 100644 --- a/config/samples/dscinitialization_v1_dscinitialization.yaml +++ b/config/samples/dscinitialization_v1_dscinitialization.yaml @@ -13,3 +13,10 @@ spec: managementState: "Managed" namespace: 'opendatahub' applicationsNamespace: 'opendatahub' + serviceMesh: + controlPlane: + metricsCollection: Istio + name: data-science-smcp + namespace: istio-system + managementState: Managed + diff --git a/controllers/datasciencecluster/datasciencecluster_controller.go b/controllers/datasciencecluster/datasciencecluster_controller.go index 8efe33f2f47..084ade96a1b 100644 --- a/controllers/datasciencecluster/datasciencecluster_controller.go +++ b/controllers/datasciencecluster/datasciencecluster_controller.go @@ -22,11 +22,11 @@ import ( "errors" "fmt" "reflect" + "strings" "time" "github.com/go-logr/logr" "github.com/hashicorp/go-multierror" - ocappsv1 "github.com/openshift/api/apps/v1" ocbuildv1 "github.com/openshift/api/build/v1" ocimgv1 "github.com/openshift/api/image/v1" v1 "github.com/openshift/api/operator/v1" @@ -35,7 +35,6 @@ import ( corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" authv1 "k8s.io/api/rbac/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" @@ -43,8 +42,10 @@ import ( "k8s.io/client-go/util/retry" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -58,7 +59,7 @@ import ( ) // DataScienceClusterReconciler reconciles a DataScienceCluster object. -type DataScienceClusterReconciler struct { +type DataScienceClusterReconciler struct { //nolint:golint,revive client.Client Scheme *runtime.Scheme Log logr.Logger @@ -69,7 +70,7 @@ type DataScienceClusterReconciler struct { } // DataScienceClusterConfig passing Spec of DSCI for reconcile DataScienceCluster. -type DataScienceClusterConfig struct { +type DataScienceClusterConfig struct { //nolint:golint,revive DSCISpec *dsci.DSCInitializationSpec } @@ -94,8 +95,7 @@ func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.R // Return and don't requeue if upgrade.HasDeleteConfigMap(r.Client) { if uninstallErr := upgrade.OperatorUninstall(r.Client, r.RestConfig); uninstallErr != nil { - return ctrl.Result{}, fmt.Errorf("error while operator uninstall: %v", - uninstallErr) + return ctrl.Result{}, fmt.Errorf("error while operator uninstall: %w", uninstallErr) } } @@ -133,7 +133,8 @@ func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.R switch len(dsciInstances.Items) { case 0: reason := status.ReconcileFailed - message := "Failed to get a valid DSCInitialization instance" + message := "Failed to get a valid DSCInitialization instance, please create a DSCI instance" + r.Log.Info(message) instance, err = r.updateStatus(ctx, instance, func(saved *dsc.DataScienceCluster) { status.SetProgressingCondition(&saved.Status.Conditions, reason, message) saved.Status.Phase = status.PhaseError @@ -142,9 +143,8 @@ func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.R r.reportError(err, instance, "failed to update DataScienceCluster condition") return ctrl.Result{}, err - } else { - return ctrl.Result{}, nil } + return ctrl.Result{}, nil case 1: dscInitializationSpec := dsciInstances.Items[0].Spec dscInitializationSpec.DeepCopyInto(r.DataScienceCluster.DSCISpec) @@ -207,14 +207,6 @@ func (r *DataScienceClusterReconciler) Reconcile(ctx context.Context, req ctrl.R } } - // Ensure all omitted components show up as explicitly disabled - instance, err = r.updateComponents(ctx, instance) - if err != nil { - _ = r.reportError(err, instance, "error updating list of components in the CR") - - return ctrl.Result{}, err - } - // Initialize error list, instead of returning errors after every component is deployed var componentErrors *multierror.Error @@ -265,6 +257,7 @@ func (r *DataScienceClusterReconciler) reconcileSubComponent(ctx context.Context component components.ComponentInterface, ) (*dsc.DataScienceCluster, error) { componentName := component.GetComponentName() + enabled := component.GetManagementState() == v1.Managed // First set conditions to reflect a component is about to be reconciled instance, err := r.updateStatus(ctx, instance, func(saved *dsc.DataScienceCluster) { @@ -280,7 +273,7 @@ func (r *DataScienceClusterReconciler) reconcileSubComponent(ctx context.Context } // Reconcile component - err = component.ReconcileComponent(r.Client, instance, r.DataScienceCluster.DSCISpec, instance.Status.InstalledComponents[componentName]) + err = component.ReconcileComponent(ctx, r.Client, r.RestConfig, instance, r.DataScienceCluster.DSCISpec, instance.Status.InstalledComponents[componentName]) if err != nil { // reconciliation failed: log errors, raise event and update status accordingly @@ -294,24 +287,23 @@ func (r *DataScienceClusterReconciler) reconcileSubComponent(ctx context.Context }) return instance, err - } else { - // reconciliation succeeded: update status accordingly - instance, err = r.updateStatus(ctx, instance, func(saved *dsc.DataScienceCluster) { - if saved.Status.InstalledComponents == nil { - saved.Status.InstalledComponents = make(map[string]bool) - } - saved.Status.InstalledComponents[componentName] = enabled - if enabled { - status.SetComponentCondition(&saved.Status.Conditions, componentName, status.ReconcileCompleted, "Component reconciled successfully", corev1.ConditionTrue) - } else { - status.RemoveComponentCondition(&saved.Status.Conditions, componentName) - } - }) - if err != nil { - instance = r.reportError(err, instance, "failed to update DataScienceCluster status after reconciling "+componentName) - - return instance, err + } + // reconciliation succeeded: update status accordingly + instance, err = r.updateStatus(ctx, instance, func(saved *dsc.DataScienceCluster) { + if saved.Status.InstalledComponents == nil { + saved.Status.InstalledComponents = make(map[string]bool) } + saved.Status.InstalledComponents[componentName] = enabled + if enabled { + status.SetComponentCondition(&saved.Status.Conditions, componentName, status.ReconcileCompleted, "Component reconciled successfully", corev1.ConditionTrue) + } else { + status.RemoveComponentCondition(&saved.Status.Conditions, componentName) + } + }) + if err != nil { + instance = r.reportError(err, instance, "failed to update DataScienceCluster status after reconciling "+componentName) + + return instance, err } return instance, nil @@ -332,36 +324,92 @@ func (r *DataScienceClusterReconciler) reportError(err error, instance *dsc.Data return instance } +var configMapPredicates = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + // Do not reconcile on prometheus configmap update, since it is handled by DSCI + if e.ObjectNew.GetName() == "prometheus" && e.ObjectNew.GetNamespace() == "redhat-ods-monitoring" { + return false + } + return true + }, +} + +// a workaround for 2.5 due to odh-model-controller serivceaccount keeps updates with label. +var saPredicates = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + if e.ObjectNew.GetName() == "odh-model-controller" && (e.ObjectNew.GetNamespace() == "redhat-ods-applications" || e.ObjectNew.GetNamespace() == "opendatahub") { + return false + } + return true + }, +} + +// a workaround for 2.5 due to modelmesh-servingruntime.serving.kserve.io keeps updates. +var modelMeshwebhookPredicates = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + return e.ObjectNew.GetName() != "modelmesh-servingruntime.serving.kserve.io" + }, +} + +var modelMeshRolePredicates = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + notAllowedNames := []string{"leader-election-role", "proxy-role", "metrics-reader", "kserve-prometheus-k8s", "odh-model-controller-role"} + for _, notallowedName := range notAllowedNames { + if e.ObjectNew.GetName() == notallowedName { + return false + } + } + return true + }, +} + +var modelMeshRBPredicates = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + notAllowedNames := []string{"leader-election-rolebinding", "proxy-rolebinding", "odh-model-controller-rolebinding-opendatahub"} + for _, notallowedName := range notAllowedNames { + if e.ObjectNew.GetName() == notallowedName { + return false + } + } + return true + }, +} + +// ignore label updates if it is from application namespace. +var modelMeshGeneralPredicates = predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + if strings.Contains(e.ObjectNew.GetName(), "odh-model-controller") || strings.Contains(e.ObjectNew.GetName(), "kserve") { + return false + } + return true + }, +} + // SetupWithManager sets up the controller with the Manager. func (r *DataScienceClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&dsc.DataScienceCluster{}). Owns(&corev1.Namespace{}). Owns(&corev1.Secret{}). - Owns(&corev1.ConfigMap{}). + Owns(&corev1.ConfigMap{}, builder.WithPredicates(configMapPredicates)). Owns(&netv1.NetworkPolicy{}). - Owns(&authv1.Role{}). - Owns(&authv1.RoleBinding{}). - Owns(&authv1.ClusterRole{}). - Owns(&authv1.ClusterRoleBinding{}). + Owns(&authv1.Role{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRolePredicates))). + Owns(&authv1.RoleBinding{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRBPredicates))). + Owns(&authv1.ClusterRole{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRolePredicates))). + Owns(&authv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshRBPredicates))). Owns(&appsv1.Deployment{}). - Owns(&appsv1.ReplicaSet{}). - Owns(&corev1.Pod{}). Owns(&corev1.PersistentVolumeClaim{}). - Owns(&corev1.Service{}). - Owns(&appsv1.DaemonSet{}). + Owns(&corev1.Service{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, modelMeshGeneralPredicates))). Owns(&appsv1.StatefulSet{}). - Owns(&ocappsv1.DeploymentConfig{}). Owns(&ocimgv1.ImageStream{}). Owns(&ocbuildv1.BuildConfig{}). - Owns(&apiextensionsv1.CustomResourceDefinition{}). Owns(&apiregistrationv1.APIService{}). Owns(&netv1.Ingress{}). Owns(&admv1.MutatingWebhookConfiguration{}). - Owns(&admv1.ValidatingWebhookConfiguration{}). - Owns(&corev1.ServiceAccount{}). + Owns(&admv1.ValidatingWebhookConfiguration{}, builder.WithPredicates(modelMeshwebhookPredicates)). + Owns(&corev1.ServiceAccount{}, builder.WithPredicates(saPredicates)). Watches(&source.Kind{Type: &dsci.DSCInitialization{}}, handler.EnqueueRequestsFromMapFunc(r.watchDataScienceClusterResources)). - Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(r.watchDataScienceClusterResources)). + Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(r.watchDataScienceClusterResources), builder.WithPredicates(configMapPredicates)). // this predicates prevents meaningless reconciliations from being triggered WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})). Complete(r) @@ -389,50 +437,35 @@ func (r *DataScienceClusterReconciler) updateStatus(ctx context.Context, origina return saved, err } -func (r *DataScienceClusterReconciler) updateComponents(ctx context.Context, original *dsc.DataScienceCluster) (*dsc.DataScienceCluster, error) { - saved := &dsc.DataScienceCluster{} - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - err := r.Client.Get(ctx, client.ObjectKeyFromObject(original), saved) - if err != nil { - return err - } - - // Try to update - err = r.Client.Update(context.TODO(), saved) - // Return err itself here (not wrapped inside another error) - // so that RetryOnConflict can identify it correctly. - return err - }) - - return saved, err -} - -func (r *DataScienceClusterReconciler) watchDataScienceClusterResources(a client.Object) (requests []reconcile.Request) { +func (r *DataScienceClusterReconciler) watchDataScienceClusterResources(a client.Object) []reconcile.Request { instanceList := &dsc.DataScienceClusterList{} err := r.Client.List(context.TODO(), instanceList) if err != nil { return nil } - if len(instanceList.Items) == 1 { - // Trigger reconcile function when uninstall configmap is created - if a.GetObjectKind().GroupVersionKind().Kind == "ConfigMap" { - labels := a.GetLabels() - if val, ok := labels[upgrade.DeleteConfigMapLabel]; ok && val == "true" { - return []reconcile.Request{{ - NamespacedName: types.NamespacedName{Name: instanceList.Items[0].Name}, - }} - } else { - return nil - } - } + var requestName string + switch { + case len(instanceList.Items) == 1: + requestName = instanceList.Items[0].Name + case len(instanceList.Items) == 0: + requestName = "default-dsc" + default: + return nil + } - return []reconcile.Request{{ - NamespacedName: types.NamespacedName{Name: instanceList.Items[0].Name}, - }} - } else if len(instanceList.Items) == 0 { - return []reconcile.Request{{ - NamespacedName: types.NamespacedName{Name: "default"}, - }} + // Trigger reconcile function when uninstall configmap is created + operatorNs, err := upgrade.GetOperatorNamespace() + if err != nil { + return nil + } + if a.GetNamespace() == operatorNs { + labels := a.GetLabels() + if val, ok := labels[upgrade.DeleteConfigMapLabel]; ok && val == "true" { + return []reconcile.Request{{ + NamespacedName: types.NamespacedName{Name: requestName}, + }} + } + return nil } return nil diff --git a/controllers/datasciencecluster/kubebuilder_rbac.go b/controllers/datasciencecluster/kubebuilder_rbac.go index 77b4fb8d323..62a56189d51 100644 --- a/controllers/datasciencecluster/kubebuilder_rbac.go +++ b/controllers/datasciencecluster/kubebuilder_rbac.go @@ -33,10 +33,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="operators.coreos.com",resources=operatorconditions,verbs=get;list;watch /* This is for operator */ -// +kubebuilder:rbac:groups="apiregistration.k8s.io",resources=apiservices,verbs=get;list;watch - -// +kubebuilder:rbac:groups="apps",resources=daemonsets,verbs=get;list;watch - // +kubebuilder:rbac:groups="operators.coreos.com",resources=catalogsources,verbs=get;list;watch // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch @@ -88,9 +84,11 @@ package datasciencecluster // +kubebuilder:rbac:groups="ray.io",resources=rayjobs,verbs=create;delete;list;update;watch;patch;get // +kubebuilder:rbac:groups="ray.io",resources=rayclusters,verbs=create;delete;list;patch;get +// +kubebuilder:rbac:groups="apiregistration.k8s.io",resources=apiservices,verbs=create;delete;list;watch;update;patch;get + // +kubebuilder:rbac:groups="operator.openshift.io",resources=consoles,verbs=list;watch;patch;delete -// +kubebuilder:rbac:groups="oauth.openshift.io",resources=oauthclients,verbs=* +// +kubebuilder:rbac:groups="oauth.openshift.io",resources=oauthclients,verbs=create;delete;list;watch;update;patch;get // +kubebuilder:rbac:groups="networking.k8s.io",resources=networkpolicies,verbs=get;create;list;watch;delete;update;patch // +kubebuilder:rbac:groups="networking.k8s.io",resources=ingresses,verbs=create;delete;list;update;watch;patch;get @@ -103,10 +101,26 @@ package datasciencecluster // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=podmonitors,verbs=get;create;delete;update;watch;list;patch // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheusrules,verbs=get;create;patch;delete;deletecollection // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheuses,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheuses/finalizers,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheuses/status,verbs=get;create;patch;delete;deletecollection -//+kubebuilder:rbac:groups=trustyai.opendatahub.io.trustyai.opendatahub.io,resources=trustyaiservices,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=trustyai.opendatahub.io.trustyai.opendatahub.io,resources=trustyaiservices/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=trustyai.opendatahub.io.trustyai.opendatahub.io,resources=trustyaiservices/finalizers,verbs=update +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=alertmanagers,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=alertmanagers/finalizers,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=alertmanagers/status,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=alertmanagerconfigs,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=thanosrulers,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=thanosrulers/finalizers,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=thanosrulers/status,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=probes,verbs=get;create;patch;delete;deletecollection +// +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheusrules,verbs=get;create;patch;delete;deletecollection + +//+kubebuilder:rbac:groups=trustyai.opendatahub.io,resources=trustyaiservices,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=trustyai.opendatahub.io,resources=trustyaiservices/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=trustyai.opendatahub.io,resources=trustyaiservices/finalizers,verbs=update + +//+kubebuilder:rbac:groups=modelregistry.opendatahub.io,resources=modelregistries,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=modelregistry.opendatahub.io,resources=modelregistries/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=modelregistry.opendatahub.io,resources=modelregistries/finalizers,verbs=update // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheuses/finalizers,verbs=get;create;patch;delete;deletecollection // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheuses/status,verbs=get;create;patch;delete;deletecollection @@ -154,7 +168,8 @@ package datasciencecluster // +kubebuilder:rbac:groups="core",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="core",resources=secrets,verbs=* +// +kubebuilder:rbac:groups="core",resources=secrets,verbs=create;delete;list;update;watch;patch;get +// +kubebuilder:rbac:groups="core",resources=secrets/finalizers,verbs=get;create;watch;update;patch;list;delete // +kubebuilder:rbac:groups="core",resources=rhmis,verbs=watch;list @@ -188,6 +203,7 @@ package datasciencecluster // +kubebuilder:rbac:groups="cert-manager.io",resources=certificates;issuers,verbs=create;patch +// OpenVino still need buildconfig // +kubebuilder:rbac:groups="build.openshift.io",resources=builds,verbs=create;patch;delete;list;watch // +kubebuilder:rbac:groups="build.openshift.io",resources=buildconfigs/instantiate,verbs=create;patch;delete;get;list;watch // +kubebuilder:rbac:groups="build.openshift.io",resources=buildconfigs,verbs=list;watch;create;patch;delete @@ -218,13 +234,10 @@ package datasciencecluster // +kubebuilder:rbac:groups="*",resources=deployments,verbs=* // +kubebuilder:rbac:groups="extensions",resources=deployments,verbs=* -// +kubebuilder:rbac:groups="apps.openshift.io",resources=deploymentconfigs,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="apps.openshift.io",resources=deploymentconfigs/instantiate,verbs=get;list;watch;create;update;patch;delete - // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch;create;patch;delete // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=validatingwebhookconfigurations,verbs=get;list;watch;create;update;delete;patch -// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=create;delete;list;update;watch;patch +// +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations,verbs=create;delete;list;update;watch;patch;get /* This is needed to derterminiate cluster type */ // +kubebuilder:rbac:groups="addons.managed.openshift.io",resources=addons,verbs=get diff --git a/controllers/datasciencecluster/suite_test.go b/controllers/datasciencecluster/suite_test.go index 92ea58d63b0..55797a332c2 100644 --- a/controllers/datasciencecluster/suite_test.go +++ b/controllers/datasciencecluster/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package datasciencecluster +package datasciencecluster_test import ( "path/filepath" diff --git a/controllers/dscinitialization/dscinitialization_controller.go b/controllers/dscinitialization/dscinitialization_controller.go index dfdd38c4c80..07bebc44776 100644 --- a/controllers/dscinitialization/dscinitialization_controller.go +++ b/controllers/dscinitialization/dscinitialization_controller.go @@ -57,7 +57,7 @@ const ( ) // DSCInitializationReconciler reconciles a DSCInitialization object. -type DSCInitializationReconciler struct { +type DSCInitializationReconciler struct { //nolint:golint,revive // Readability client.Client Scheme *runtime.Scheme Log logr.Logger @@ -69,10 +69,11 @@ type DSCInitializationReconciler struct { // +kubebuilder:rbac:groups="dscinitialization.opendatahub.io",resources=dscinitializations/finalizers,verbs=get;update;patch;delete // +kubebuilder:rbac:groups="dscinitialization.opendatahub.io",resources=dscinitializations,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="features.opendatahub.io",resources=featuretrackers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="features.opendatahub.io",resources=featuretrackers/status,verbs=get;update;patch;delete // +kubebuilder:rbac:groups="kfdef.apps.kubeflow.org",resources=kfdefs,verbs=get;list;watch;create;update;patch;delete // Reconcile contains controller logic specific to DSCInitialization instance updates. -func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { //nolint +func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { //nolint:funlen,gocyclo,maintidx r.Log.Info("Reconciling DSCInitialization.", "DSCInitialization Request.Name", req.Name) instances := &dsciv1.DSCInitializationList{} @@ -90,11 +91,26 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re case len(instances.Items) == 1: instance = &instances.Items[0] case len(instances.Items) > 1: - message := fmt.Sprintf("only one instance of DSCInitialization object is allowed. Update existing instance name %s", req.Name) - _, _ = r.updateStatus(ctx, instance, func(saved *dsciv1.DSCInitialization) { - status.SetErrorCondition(&saved.Status.Conditions, status.DuplicateDSCInitialization, message) - saved.Status.Phase = status.PhaseError - }) + // find out the one by created timestamp and use it as the default one + earliestDSCI := &instances.Items[0] + for _, instance := range instances.Items { + currentDSCI := instance + if currentDSCI.CreationTimestamp.Before(&earliestDSCI.CreationTimestamp) { + earliestDSCI = ¤tDSCI + } + } + message := fmt.Sprintf("only one instance of DSCInitialization object is allowed. Please delete other instances than %s", earliestDSCI.Name) + // update all instances Message and Status + for _, deletionInstance := range instances.Items { + deletionInstance := deletionInstance + if deletionInstance.Name != earliestDSCI.Name { + _, _ = r.updateStatus(ctx, &deletionInstance, func(saved *dsciv1.DSCInitialization) { + status.SetErrorCondition(&saved.Status.Conditions, status.DuplicateDSCInitialization, message) + saved.Status.Phase = status.PhaseError + }) + } + } + return ctrl.Result{}, errors.New(message) } @@ -187,6 +203,15 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re return ctrl.Result{}, nil default: + // Check namespace is not exist, then create + namespace := instance.Spec.ApplicationsNamespace + r.Log.Info("Standard Reconciling workflow to create namespaces") + err = r.createOdhNamespace(ctx, instance, namespace) + if err != nil { + // no need to log error as it was already logged in createOdhNamespace + return reconcile.Result{}, err + } + // Start reconciling if instance.Status.Conditions == nil { reason := status.ReconcileInit @@ -204,22 +229,6 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re } } - // Check namespace is not exist, then create - namespace := instance.Spec.ApplicationsNamespace - r.Log.Info("Standard Reconciling workflow to create namespaces") - if err = r.createOdhNamespace(ctx, instance, namespace); err != nil { - // no need to log error as it was already logged in createOdhNamespace - return reconcile.Result{}, err - } - - // Apply update from legacy operator - // TODO: Update upgrade logic to get components through KfDef - if err = upgrade.UpdateFromLegacyVersion(r.Client, platform); err != nil { - r.Log.Error(err, "unable to update from legacy operator version") - - return reconcile.Result{}, err - } - switch platform { case deploy.SelfManagedRhods: err := r.createUserGroup(ctx, instance, "rhods-admins") @@ -297,8 +306,6 @@ func (r *DSCInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&authv1.ClusterRole{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). Owns(&authv1.ClusterRoleBinding{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). Owns(&appsv1.Deployment{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). - Owns(&appsv1.ReplicaSet{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). - Owns(&corev1.Pod{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). Owns(&corev1.ServiceAccount{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). Owns(&corev1.Service{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). Owns(&routev1.Route{}, builder.WithPredicates(predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{}))). @@ -328,8 +335,8 @@ func (r *DSCInitializationReconciler) updateStatus(ctx context.Context, original var SecretContentChangedPredicate = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldSecret := e.ObjectOld.(*corev1.Secret) - newSecret := e.ObjectNew.(*corev1.Secret) + oldSecret, _ := e.ObjectOld.(*corev1.Secret) + newSecret, _ := e.ObjectNew.(*corev1.Secret) return !reflect.DeepEqual(oldSecret.Data, newSecret.Data) }, @@ -337,8 +344,8 @@ var SecretContentChangedPredicate = predicate.Funcs{ var CMContentChangedPredicate = predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldCM := e.ObjectOld.(*corev1.ConfigMap) - newCM := e.ObjectNew.(*corev1.ConfigMap) + oldCM, _ := e.ObjectOld.(*corev1.ConfigMap) + newCM, _ := e.ObjectNew.(*corev1.ConfigMap) return !reflect.DeepEqual(oldCM.Data, newCM.Data) }, @@ -351,17 +358,16 @@ var DSCDeletionPredicate = predicate.Funcs{ }, } -func (r *DSCInitializationReconciler) watchMonitoringConfigMapResource(a client.Object) (requests []reconcile.Request) { +func (r *DSCInitializationReconciler) watchMonitoringConfigMapResource(a client.Object) []reconcile.Request { if a.GetName() == "prometheus" && a.GetNamespace() == "redhat-ods-monitoring" { r.Log.Info("Found monitoring configmap has updated, start reconcile") return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: "prometheus", Namespace: "redhat-ods-monitoring"}}} - } else { - return nil } + return nil } -func (r *DSCInitializationReconciler) watchMonitoringSecretResource(a client.Object) (requests []reconcile.Request) { +func (r *DSCInitializationReconciler) watchMonitoringSecretResource(a client.Object) []reconcile.Request { operatorNs, err := upgrade.GetOperatorNamespace() if err != nil { return nil @@ -370,12 +376,11 @@ func (r *DSCInitializationReconciler) watchMonitoringSecretResource(a client.Obj r.Log.Info("Found monitoring secret has updated, start reconcile") return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: "addon-managed-odh-parameters", Namespace: operatorNs}}} - } else { - return nil } + return nil } -func (r *DSCInitializationReconciler) watchDSCResource(_ client.Object) (requests []reconcile.Request) { +func (r *DSCInitializationReconciler) watchDSCResource(_ client.Object) []reconcile.Request { instanceList := &dscv1.DataScienceClusterList{} if err := r.Client.List(context.TODO(), instanceList); err != nil { // do not handle if cannot get list diff --git a/controllers/dscinitialization/dscinitialization_test.go b/controllers/dscinitialization/dscinitialization_test.go index 182cd5411e8..c2588919cc3 100644 --- a/controllers/dscinitialization/dscinitialization_test.go +++ b/controllers/dscinitialization/dscinitialization_test.go @@ -111,7 +111,7 @@ var _ = Describe("DataScienceCluster initialization", func() { Eventually(dscInitializationIsReady(applicationName, workingNamespace, foundDsci), timeout, interval).Should(BeTrue()) // then foundMonitoringNamespace := &corev1.Namespace{} - Eventually(Eventually(namespaceExists(monitoringNamespace2, foundMonitoringNamespace), timeout, interval).Should(BeTrue()), timeout, interval).Should(BeTrue()) + Eventually(namespaceExists(monitoringNamespace2, foundMonitoringNamespace), timeout, interval).Should(BeTrue()) Expect(foundMonitoringNamespace.Name).Should(Equal(monitoringNamespace2)) }) }) @@ -228,7 +228,6 @@ var _ = Describe("DataScienceCluster initialization", func() { }) }) -// cleanup utility func. func cleanupResources() { defaultNamespace := client.InNamespace(workingNamespace) appNamespace := client.InNamespace(applicationNamespace) @@ -263,7 +262,7 @@ func namespaceExists(ns string, obj client.Object) func() bool { } } -func objectExists(ns string, name string, obj client.Object) func() bool { //nolint +func objectExists(ns string, name string, obj client.Object) func() bool { //nolint:unparam return func() bool { err := k8sClient.Get(context.Background(), client.ObjectKey{Name: ns, Namespace: name}, obj) @@ -291,7 +290,7 @@ func createDSCI(appName string, enableMonitoring operatorv1.ManagementState, mon } } -func dscInitializationIsReady(ns string, name string, dsciObj *dsci.DSCInitialization) func() bool { //nolint +func dscInitializationIsReady(ns string, name string, dsciObj *dsci.DSCInitialization) func() bool { //nolint:unparam return func() bool { _ = k8sClient.Get(context.Background(), client.ObjectKey{Name: ns, Namespace: name}, dsciObj) diff --git a/controllers/dscinitialization/monitoring.go b/controllers/dscinitialization/monitoring.go index cb101c70298..530add5470b 100644 --- a/controllers/dscinitialization/monitoring.go +++ b/controllers/dscinitialization/monitoring.go @@ -37,7 +37,7 @@ var ( ) // only when reconcile on DSCI CR, initial set to true -// if reconcile from monitoring, initial set to false, skip blackbox and rolebinding +// if reconcile from monitoring, initial set to false, skip blackbox and rolebinding. func (r *DSCInitializationReconciler) configureManagedMonitoring(ctx context.Context, dscInit *dsci.DSCInitialization, initial string) error { if initial == "init" { // configure Blackbox exporter @@ -46,9 +46,19 @@ func (r *DSCInitializationReconciler) configureManagedMonitoring(ctx context.Con } } if initial == "revertbackup" { + // TODO: implement with a better solution + // to have - before component name is to filter out the real rules file line + // e.g line of "workbenches-recording.rules: |" err := common.MatchLineInFile(filepath.Join(prometheusConfigPath, "prometheus-configs.yaml"), map[string]string{ - "*.rules: ": "", + "(.*)-(.*)workbenches(.*).rules": "", + "(.*)-(.*)rhods-dashboard(.*).rules": "", + "(.*)-(.*)codeflare(.*).rules": "", + "(.*)-(.*)data-science-pipelines-operator(.*).rules": "", + "(.*)-(.*)model-mesh(.*).rules": "", + "(.*)-(.*)odh-model-controller(.*).rules": "", + "(.*)-(.*)ray(.*).rules": "", + "(.*)-(.*)trustyai(.*).rules": "", }) if err != nil { r.Log.Error(err, "error to remove previous enabled component rules") @@ -122,7 +132,7 @@ func configureAlertManager(ctx context.Context, dsciInit *dsci.DSCInitialization // special handling for dev-mod consolelinkDomain, err := common.GetDomain(r.Client, NameConsoleLink, NamespaceConsoleLink) if err != nil { - return fmt.Errorf("error getting console route URL : %v", err) + return fmt.Errorf("error getting console route URL : %w", err) } if strings.Contains(consolelinkDomain, "devshift.org") { r.Log.Info("inject alertmanage-configs.yaml for dev mode1") @@ -199,18 +209,17 @@ func configurePrometheus(ctx context.Context, dsciInit *dsci.DSCInitialization, // Update prometheus-config for dashboard, dsp and workbench consolelinkDomain, err := common.GetDomain(r.Client, NameConsoleLink, NamespaceConsoleLink) if err != nil { - return fmt.Errorf("error getting console route URL : %v", err) - } else { - err = common.ReplaceStringsInFile(filepath.Join(prometheusConfigPath, "prometheus-configs.yaml"), - map[string]string{ - "": dsciInit.Spec.ApplicationsNamespace, - "": dsciInit.Spec.Monitoring.Namespace, - "": consolelinkDomain, - }) - if err != nil { - r.Log.Error(err, "error to inject data to prometheus-configs.yaml") - return err - } + return fmt.Errorf("error getting console route URL : %w", err) + } + err = common.ReplaceStringsInFile(filepath.Join(prometheusConfigPath, "prometheus-configs.yaml"), + map[string]string{ + "": dsciInit.Spec.ApplicationsNamespace, + "": dsciInit.Spec.Monitoring.Namespace, + "": consolelinkDomain, + }) + if err != nil { + r.Log.Error(err, "error to inject data to prometheus-configs.yaml") + return err } // Deploy prometheus manifests from prometheus/apps @@ -314,7 +323,7 @@ func configurePrometheus(ctx context.Context, dsciInit *dsci.DSCInitialization, if len(existingPromDep.Spec.Template.Spec.InitContainers) > 0 { err = r.Client.Delete(context.TODO(), existingPromDep) if err != nil { - return fmt.Errorf("error deleting legacy prometheus deployment %v", err) + return fmt.Errorf("error deleting legacy prometheus deployment %w", err) } } @@ -357,7 +366,7 @@ func configureBlackboxExporter(ctx context.Context, dsciInit *dsci.DSCInitializa if len(existingBlackboxExp.Spec.Template.Spec.InitContainers) > 0 { err = r.Client.Delete(context.TODO(), existingBlackboxExp) if err != nil { - return fmt.Errorf("error deleting legacy blackbox deployment %v", err) + return fmt.Errorf("error deleting legacy blackbox deployment %w", err) } } @@ -369,7 +378,7 @@ func configureBlackboxExporter(ctx context.Context, dsciInit *dsci.DSCInitializa dsciInit.Spec.Monitoring.Namespace, "blackbox-exporter", dsciInit.Spec.Monitoring.ManagementState == operatorv1.Managed); err != nil { - r.Log.Error(err, "error to deploy manifests: %w", err) + r.Log.Error(err, "error to deploy manifests: %w", "error", err) return err } } else { @@ -379,7 +388,7 @@ func configureBlackboxExporter(ctx context.Context, dsciInit *dsci.DSCInitializa dsciInit.Spec.Monitoring.Namespace, "blackbox-exporter", dsciInit.Spec.Monitoring.ManagementState == operatorv1.Managed); err != nil { - r.Log.Error(err, "error to deploy manifests: %w", err) + r.Log.Error(err, "error to deploy manifests: %w", "error", err) return err } } diff --git a/controllers/dscinitialization/servicemesh_setup.go b/controllers/dscinitialization/servicemesh_setup.go index 7bfca96a221..f976652fc3a 100644 --- a/controllers/dscinitialization/servicemesh_setup.go +++ b/controllers/dscinitialization/servicemesh_setup.go @@ -2,7 +2,6 @@ package dscinitialization import ( "path" - "path/filepath" operatorv1 "github.com/openshift/api/operator/v1" corev1 "k8s.io/api/core/v1" @@ -12,23 +11,21 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" ) -const templatesDir = "templates/servicemesh" - func (r *DSCInitializationReconciler) configureServiceMesh(instance *dsciv1.DSCInitialization) error { - if instance.Spec.ServiceMesh.ManagementState == operatorv1.Managed { - serviceMeshInitializer := feature.NewFeaturesInitializer(&instance.Spec, configureServiceMeshFeatures) - - if err := serviceMeshInitializer.Prepare(); err != nil { - r.Log.Error(err, "failed configuring service mesh resources") - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "failed configuring service mesh resources") - - return err - } + switch instance.Spec.ServiceMesh.ManagementState { + case operatorv1.Managed: + serviceMeshFeatures := feature.ClusterFeaturesHandler(instance, configureServiceMeshFeatures()) - if err := serviceMeshInitializer.Apply(); err != nil { + if err := serviceMeshFeatures.Apply(); err != nil { r.Log.Error(err, "failed applying service mesh resources") r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "failed applying service mesh resources") - + return err + } + case operatorv1.Unmanaged: + r.Log.Info("ServiceMesh CR is not configured by the operator, we won't do anything") + case operatorv1.Removed: + r.Log.Info("existing ServiceMesh CR (owned by operator) will be removed") + if err := r.removeServiceMesh(instance); err != nil { return err } } @@ -37,17 +34,11 @@ func (r *DSCInitializationReconciler) configureServiceMesh(instance *dsciv1.DSCI } func (r *DSCInitializationReconciler) removeServiceMesh(instance *dsciv1.DSCInitialization) error { + // on condition of Managed, do not handle Removed when set to Removed it trigger DSCI reconcile to cleanup if instance.Spec.ServiceMesh.ManagementState == operatorv1.Managed { - serviceMeshInitializer := feature.NewFeaturesInitializer(&instance.Spec, configureServiceMeshFeatures) - - if err := serviceMeshInitializer.Prepare(); err != nil { - r.Log.Error(err, "failed configuring service mesh resources") - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "failed configuring service mesh resources") - - return err - } + serviceMeshFeatures := feature.ClusterFeaturesHandler(instance, configureServiceMeshFeatures()) - if err := serviceMeshInitializer.Delete(); err != nil { + if err := serviceMeshFeatures.Delete(); err != nil { r.Log.Error(err, "failed deleting service mesh resources") r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "failed deleting service mesh resources") @@ -58,47 +49,43 @@ func (r *DSCInitializationReconciler) removeServiceMesh(instance *dsciv1.DSCInit return nil } -func configureServiceMeshFeatures(s *feature.FeaturesInitializer) error { - var rootDir = filepath.Join(feature.BaseOutputDir, s.DSCInitializationSpec.ApplicationsNamespace) - if err := feature.CopyEmbeddedFiles(templatesDir, rootDir); err != nil { - return err - } - - serviceMeshSpec := s.ServiceMesh +func configureServiceMeshFeatures() feature.FeaturesProvider { + return func(handler *feature.FeaturesHandler) error { + serviceMeshSpec := handler.DSCInitializationSpec.ServiceMesh - smcpCreation, errSmcp := feature.CreateFeature("mesh-control-plane-creation"). - For(s.DSCInitializationSpec). - Manifests( - path.Join(rootDir, templatesDir, "/base"), - ). - PreConditions( - servicemesh.EnsureServiceMeshOperatorInstalled, - feature.CreateNamespaceIfNotExists(serviceMeshSpec.ControlPlane.Namespace), - ). - PostConditions( - feature.WaitForPodsToBeReady(serviceMeshSpec.ControlPlane.Namespace), - ). - Load() - if errSmcp != nil { - return errSmcp - } - s.Features = append(s.Features, smcpCreation) - - if serviceMeshSpec.ControlPlane.MetricsCollection == "Istio" { - metricsCollection, errMetrics := feature.CreateFeature("mesh-metrics-collection"). - For(s.DSCInitializationSpec). + smcpCreationErr := feature.CreateFeature("mesh-control-plane-creation"). + For(handler). Manifests( - path.Join(rootDir, templatesDir, "metrics-collection"), + path.Join(feature.ServiceMeshDir, "base", "create-smcp.tmpl"), ). PreConditions( - servicemesh.EnsureServiceMeshInstalled, + servicemesh.EnsureServiceMeshOperatorInstalled, + feature.CreateNamespaceIfNotExists(serviceMeshSpec.ControlPlane.Namespace), + ). + PostConditions( + feature.WaitForPodsToBeReady(serviceMeshSpec.ControlPlane.Namespace), ). Load() - if errMetrics != nil { - return errMetrics + + if smcpCreationErr != nil { + return smcpCreationErr } - s.Features = append(s.Features, metricsCollection) - } - return nil + if serviceMeshSpec.ControlPlane.MetricsCollection == "Istio" { + metricsCollectionErr := feature.CreateFeature("mesh-metrics-collection"). + For(handler). + PreConditions( + servicemesh.EnsureServiceMeshInstalled, + ). + Manifests( + path.Join(feature.ServiceMeshDir, "metrics-collection"), + ). + Load() + if metricsCollectionErr != nil { + return metricsCollectionErr + } + } + + return nil + } } diff --git a/controllers/dscinitialization/utils.go b/controllers/dscinitialization/utils.go index c944cd8ebe6..fc87113204e 100644 --- a/controllers/dscinitialization/utils.go +++ b/controllers/dscinitialization/utils.go @@ -35,8 +35,12 @@ var ( // - Pod security labels for baseline permissions // - ConfigMap 'odh-common-config' // - Network Policies 'opendatahub' that allow traffic between the ODH namespaces -// - RoleBinding 'opendatahub' +// - RoleBinding 'opendatahub'. func (r *DSCInitializationReconciler) createOdhNamespace(ctx context.Context, dscInit *dsci.DSCInitialization, name string) error { + platform, err := deploy.GetPlatform(r.Client) + if err != nil { + return err + } // Expected namespace for the given name desiredNamespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ @@ -50,7 +54,7 @@ func (r *DSCInitializationReconciler) createOdhNamespace(ctx context.Context, ds // Create Namespace if it doesn't exist foundNamespace := &corev1.Namespace{} - err := r.Get(ctx, client.ObjectKey{Name: name}, foundNamespace) + err = r.Get(ctx, client.ObjectKey{Name: name}, foundNamespace) if err != nil { if apierrs.IsNotFound(err) { r.Log.Info("Creating namespace", "name", name) @@ -71,7 +75,7 @@ func (r *DSCInitializationReconciler) createOdhNamespace(ctx context.Context, ds } } else if dscInit.Spec.Monitoring.ManagementState == operatorv1.Managed { r.Log.Info("Patching application namespace for Managed cluster", "name", name) - labelPatch := `{"metadata":{"labels":{"openshift.io/cluster-monitoring":"true","pod-security.kubernetes.io/enforce":"baseline","opendatahub.io/generated-namespace": "true"}}}` //nolint + labelPatch := `{"metadata":{"labels":{"openshift.io/cluster-monitoring":"true","pod-security.kubernetes.io/enforce":"baseline","opendatahub.io/generated-namespace": "true"}}}` err = r.Patch(ctx, foundNamespace, client.RawPatch(types.MergePatchType, []byte(labelPatch))) if err != nil { @@ -106,7 +110,7 @@ func (r *DSCInitializationReconciler) createOdhNamespace(ctx context.Context, ds return err } } else { // force to patch monitoring namespace with label for cluster-monitoring - r.Log.Info("Patching monitoring namespace for Managed cluster", "name", monitoringName) + r.Log.Info("Patching monitoring namespace", "name", monitoringName) labelPatch := `{"metadata":{"labels":{"openshift.io/cluster-monitoring":"true", "pod-security.kubernetes.io/enforce":"baseline","opendatahub.io/generated-namespace": "true"}}}` err = r.Patch(ctx, foundMonitoringNamespace, client.RawPatch(types.MergePatchType, []byte(labelPatch))) @@ -118,26 +122,21 @@ func (r *DSCInitializationReconciler) createOdhNamespace(ctx context.Context, ds // Patch downstream Operator Namespace if it is monitoring enabled if dscInit.Spec.Monitoring.ManagementState == operatorv1.Managed { - platform, err := deploy.GetPlatform(r.Client) - if err != nil { - return err - } if platform == deploy.ManagedRhods || platform == deploy.SelfManagedRhods { operatorNs, err := upgrade.GetOperatorNamespace() if err != nil { r.Log.Error(err, "error getting operator namespace") return err } - r.Log.Info("Patching operator namespace for Managed cluster", "name", operatorNs) + r.Log.Info("Patching operator namespace", "name", operatorNs) labelPatch := `{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"baseline"}}}` operatorNamespace := &corev1.Namespace{} if err := r.Get(ctx, client.ObjectKey{Name: operatorNs}, operatorNamespace); err != nil { return err - } else { - err = r.Patch(ctx, operatorNamespace, client.RawPatch(types.MergePatchType, []byte(labelPatch))) - if err != nil { - return err - } + } + err = r.Patch(ctx, operatorNamespace, client.RawPatch(types.MergePatchType, []byte(labelPatch))) + if err != nil { + return err } } } @@ -265,12 +264,12 @@ func (r *DSCInitializationReconciler) reconcileDefaultNetworkPolicy(ctx context. }, }, }, - { // OR logic for ROSA + { // OR logic From: []netv1.NetworkPolicyPeer{ - { // need this to access dashboard + { // need this for access dashboard NamespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "kubernetes.io/metadata.name": "openshift-ingress", + "network.openshift.io/policy-group": "ingress", }, }, }, @@ -355,8 +354,8 @@ func CompareNotebookNetworkPolicies(np1 netv1.NetworkPolicy, np2 netv1.NetworkPo func (r *DSCInitializationReconciler) waitForManagedSecret(ctx context.Context, name string, namespace string) (*corev1.Secret, error) { managedSecret := &corev1.Secret{} - err := wait.PollUntilContextTimeout(ctx, resourceInterval, resourceTimeout, false, func(ctx context.Context) (done bool, err error) { - err = r.Client.Get(ctx, client.ObjectKey{ + err := wait.PollUntilContextTimeout(ctx, resourceInterval, resourceTimeout, false, func(ctx context.Context) (bool, error) { + err := r.Client.Get(ctx, client.ObjectKey{ Namespace: namespace, Name: name, }, managedSecret) @@ -366,9 +365,8 @@ func (r *DSCInitializationReconciler) waitForManagedSecret(ctx context.Context, return false, nil } return false, err - } else { - return true, nil } + return true, nil }) return managedSecret, err @@ -382,8 +380,7 @@ func GenerateRandomHex(length int) ([]byte, error) { randomBytes := make([]byte, numBytes) // Read random bytes from the crypto/rand source - _, err := rand.Read(randomBytes) - if err != nil { + if _, err := rand.Read(randomBytes); err != nil { return nil, err } diff --git a/controllers/secretgenerator/secret.go b/controllers/secretgenerator/secret.go index fe91f42ea37..ddc45f6a7e5 100644 --- a/controllers/secretgenerator/secret.go +++ b/controllers/secretgenerator/secret.go @@ -8,12 +8,13 @@ import ( "strconv" ) +//nolint:golint,revive,stylecheck //CAPS is preferred for const const ( - SECRET_NAME_ANNOTATION = "secret-generator.opendatahub.io/name" //nolint - SECRET_TYPE_ANNOTATION = "secret-generator.opendatahub.io/type" //nolint - SECRET_LENGTH_ANNOTATION = "secret-generator.opendatahub.io/complexity" //nolint - SECRET_OAUTH_CLIENT_ANNOTATION = "secret-generator.opendatahub.io/oauth-client-route" //nolint - SECRET_DEFAULT_COMPLEXITY = 16 //nolint + SECRET_NAME_ANNOTATION = "secret-generator.opendatahub.io/name" + SECRET_TYPE_ANNOTATION = "secret-generator.opendatahub.io/type" + SECRET_LENGTH_ANNOTATION = "secret-generator.opendatahub.io/complexity" + SECRET_OAUTH_CLIENT_ANNOTATION = "secret-generator.opendatahub.io/oauth-client-route" + SECRET_DEFAULT_COMPLEXITY = 16 letterRunes = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" diff --git a/controllers/secretgenerator/secret_test.go b/controllers/secretgenerator/secret_test.go index 240986bb974..2d6b5dfb842 100644 --- a/controllers/secretgenerator/secret_test.go +++ b/controllers/secretgenerator/secret_test.go @@ -1,14 +1,23 @@ -package secretgenerator +package secretgenerator_test import ( "errors" "testing" + + "github.com/opendatahub-io/opendatahub-operator/v2/controllers/secretgenerator" +) + +const ( + errEmptyAnnotation = "secret annotations is empty" + errNameAnnotationNotFound = "name annotation not found in secret" + errTypeAnnotationNotFound = "type annotation not found in secret" + errUnsupportedType = "secret type is not supported" ) func TestNewSecret(t *testing.T) { cases := map[string]struct { annotations map[string]string - secret Secret + secret secretgenerator.Secret err error }{ "Annotations are not defined": { @@ -39,10 +48,10 @@ func TestNewSecret(t *testing.T) { "secret-generator.opendatahub.io/name": "example", "secret-generator.opendatahub.io/type": "random", }, - secret: Secret{ + secret: secretgenerator.Secret{ Name: "example", Type: "random", - Complexity: SECRET_DEFAULT_COMPLEXITY, + Complexity: secretgenerator.SECRET_DEFAULT_COMPLEXITY, }, }, "Generate a random string secret with custom complexity": { @@ -51,7 +60,7 @@ func TestNewSecret(t *testing.T) { "secret-generator.opendatahub.io/type": "random", "secret-generator.opendatahub.io/complexity": "128", }, - secret: Secret{ + secret: secretgenerator.Secret{ Name: "example", Type: "random", Complexity: 128, @@ -62,10 +71,10 @@ func TestNewSecret(t *testing.T) { "secret-generator.opendatahub.io/name": "example", "secret-generator.opendatahub.io/type": "oauth", }, - secret: Secret{ + secret: secretgenerator.Secret{ Name: "example", Type: "oauth", - Complexity: SECRET_DEFAULT_COMPLEXITY, + Complexity: secretgenerator.SECRET_DEFAULT_COMPLEXITY, }, }, "Generate an OAuth secret with custom complexity": { @@ -74,7 +83,7 @@ func TestNewSecret(t *testing.T) { "secret-generator.opendatahub.io/type": "oauth", "secret-generator.opendatahub.io/complexity": "24", }, - secret: Secret{ + secret: secretgenerator.Secret{ Name: "example", Type: "oauth", Complexity: 24, @@ -83,8 +92,9 @@ func TestNewSecret(t *testing.T) { } for name, tc := range cases { + tc := tc t.Run(name, func(t *testing.T) { - secret, err := NewSecretFrom(tc.annotations) + secret, err := secretgenerator.NewSecretFrom(tc.annotations) if err != nil { if err.Error() != tc.err.Error() { t.Errorf("Expected error: %v, got: %v\n", diff --git a/controllers/secretgenerator/secretgenerator_controller.go b/controllers/secretgenerator/secretgenerator_controller.go index 8960de68c68..80df0502a67 100644 --- a/controllers/secretgenerator/secretgenerator_controller.go +++ b/controllers/secretgenerator/secretgenerator_controller.go @@ -49,13 +49,8 @@ const ( var secGenLog = log.Log.WithName("secret-generator") -// +kubebuilder:rbac:groups="oauth.openshift.io",resources=oauthclients,verbs=create;delete;get -// +kubebuilder:rbac:groups="core",resources=secrets,verbs=watch;get;create -// +kubebuilder:rbac:groups="route.openshift.io",resources=routes,verbs=get -// +kubebuilder:rbac:groups="core",resources=secrets/finalizers,verbs=* - // SecretGeneratorReconciler holds the controller configuration. -type SecretGeneratorReconciler struct { +type SecretGeneratorReconciler struct { //nolint:golint,revive // Readability Client client.Client Scheme *runtime.Scheme } @@ -76,6 +71,8 @@ func (r *SecretGeneratorReconciler) SetupWithManager(mgr ctrl.Manager) error { GenericFunc: func(e event.GenericEvent) bool { return false }, + // this only watch for secret deletion if has with annotation + // e.g dashboard-oauth-client but not dashboard-oauth-client-generated DeleteFunc: func(e event.DeleteEvent) bool { if _, found := e.Object.GetAnnotations()[SECRET_NAME_ANNOTATION]; found { return true @@ -141,8 +138,7 @@ func (r *SecretGeneratorReconciler) Reconcile(ctx context.Context, request ctrl. secret, err := NewSecretFrom(foundSecret.GetAnnotations()) if err != nil { - secGenLog.Error(err, "error creating secret") - + secGenLog.Error(err, "error creating secret %s in %s", generatedSecret.Name, generatedSecret.Namespace) return ctrl.Result{}, err } @@ -154,16 +150,18 @@ func (r *SecretGeneratorReconciler) Reconcile(ctx context.Context, request ctrl. if err != nil { return ctrl.Result{}, err } + secGenLog.Info("Done generating secret in namespace", + "secret", generatedSecret.Name, "namespace", generatedSecret.Namespace) + // check if annotation oauth-client-route exists if secret.OAuthClientRoute != "" { // Get OauthClient Route oauthClientRoute, err := r.getRoute(ctx, secret.OAuthClientRoute, request.Namespace) if err != nil { - secGenLog.Error(err, "Unable to retrieve route", "route-name", secret.OAuthClientRoute) - + secGenLog.Error(err, "Unable to retrieve route from OAuthClient", "route-name", secret.OAuthClientRoute) return ctrl.Result{}, err } // Generate OAuthClient for the generated secret - secGenLog.Info("Generating an oauth client resource for route", "route-name", oauthClientRoute.Name) + secGenLog.Info("Generating an OAuthClient CR for route", "route-name", oauthClientRoute.Name) err = r.createOAuthClient(ctx, foundSecret.Name, secret.Value, oauthClientRoute.Spec.Host) if err != nil { secGenLog.Error(err, "error creating oauth client resource. Recreate the Secret", "secret-name", @@ -185,8 +183,8 @@ func (r *SecretGeneratorReconciler) Reconcile(ctx context.Context, request ctrl. func (r *SecretGeneratorReconciler) getRoute(ctx context.Context, name string, namespace string) (*routev1.Route, error) { route := &routev1.Route{} // Get spec.host from route - err := wait.PollUntilContextTimeout(ctx, resourceRetryInterval, resourceRetryTimeout, false, func(ctx context.Context) (done bool, err error) { - err = r.Client.Get(ctx, client.ObjectKey{ + err := wait.PollUntilContextTimeout(ctx, resourceRetryInterval, resourceRetryTimeout, false, func(ctx context.Context) (bool, error) { + err := r.Client.Get(ctx, client.ObjectKey{ Name: name, Namespace: namespace, }, route) @@ -199,9 +197,8 @@ func (r *SecretGeneratorReconciler) getRoute(ctx context.Context, name string, n } if route.Spec.Host == "" { return false, nil - } else { - return true, nil } + return true, nil }) if err != nil { return nil, err @@ -215,7 +212,7 @@ func (r *SecretGeneratorReconciler) createOAuthClient(ctx context.Context, name oauthClient := &ocv1.OAuthClient{ TypeMeta: metav1.TypeMeta{ Kind: "OAuthClient", - APIVersion: "v1", + APIVersion: "oauth.openshift.io/v1", }, ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -231,12 +228,11 @@ func (r *SecretGeneratorReconciler) createOAuthClient(ctx context.Context, name secGenLog.Info("OAuth client resource already exists, patch it", "name", oauthClient.Name) data, err := json.Marshal(oauthClient) if err != nil { - return fmt.Errorf("failed to get DataScienceCluster custom resource data: %v", err) + return fmt.Errorf("failed to get DataScienceCluster custom resource data: %w", err) } - err = r.Client.Patch(context.TODO(), oauthClient, client.RawPatch(types.ApplyPatchType, data), - client.ForceOwnership, client.FieldOwner("opendatahub-operator")) - if err != nil { - return err + if err = r.Client.Patch(context.TODO(), oauthClient, client.RawPatch(types.ApplyPatchType, data), + client.ForceOwnership, client.FieldOwner("rhods-operator")); err != nil { + return fmt.Errorf("failed to patch existing OAuthClient CR: %w", err) } return nil } @@ -259,9 +255,8 @@ func (r *SecretGeneratorReconciler) deleteOAuthClient(ctx context.Context, secre return err } - err = r.Client.Delete(ctx, oauthClient) - if err != nil { - return fmt.Errorf("error deleting OAuthClient %v", oauthClient.Name) + if err = r.Client.Delete(ctx, oauthClient); err != nil { + return fmt.Errorf("error deleting OAuthClient %s: %w", oauthClient.Name, err) } return nil diff --git a/docs/DESIGN.md b/docs/DESIGN.md index d8ce2237d6b..fb2b57419bf 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -53,13 +53,15 @@ To deploy ODH components seamlessly, ODH operator will watch two CRDs: datasciencepipelines: managementState: Managed kserve: - managementState: Removed - modelmeshserving: managementState: Managed + modelmeshserving: + managementState: Removed ray: managementState: Removed workbenches: managementState: Managed + trustyai: + managementState: Managed ``` 2. Enable only Dashboard and Workbenches(Jupyter Notebooks) diff --git a/docs/Dev-Preview.md b/docs/Dev-Preview.md index 67fe3d31bbf..c0586dfc3cd 100644 --- a/docs/Dev-Preview.md +++ b/docs/Dev-Preview.md @@ -86,7 +86,7 @@ metadata: spec: components: codeflare: - managementState: Managed + managementState: Removed dashboard: managementState: Managed datasciencepipelines: @@ -94,11 +94,13 @@ spec: kserve: managementState: Managed modelmeshserving: - managementState: Managed + managementState: Removed ray: - managementState: Managed + managementState: Removed workbenches: managementState: Managed + trustyai: + managementState: Managed EOF ``` diff --git a/get_all_manifests.sh b/get_all_manifests.sh index ef8807326de..00df995a331 100755 --- a/get_all_manifests.sh +++ b/get_all_manifests.sh @@ -3,11 +3,12 @@ set -e GITHUB_URL="https://github.com/" -# component: notebook, dsp, kserve, dashbaord, cf/ray, trustyai, modelmesh. +# component: notebook, dsp, kserve, dashbaord, cf/ray/kueue, trustyai, modelmesh, modelregistry. # in the format of "repo-org:repo-name:branch-name:source-folder:target-folder". declare -A COMPONENT_MANIFESTS=( ["codeflare"]="opendatahub-io:codeflare-operator:main:config:codeflare" ["ray"]="opendatahub-io:kuberay:master:ray-operator/config:ray" + ["kueue"]="opendatahub-io:kueue:dev:config:kueue" ["data-science-pipelines-operator"]="opendatahub-io:data-science-pipelines-operator:main:config:data-science-pipelines-operator" ["odh-dashboard"]="opendatahub-io:odh-dashboard:incubation:manifests:dashboard" ["kf-notebook-controller"]="opendatahub-io:kubeflow:v1.7-branch:components/notebook-controller/config:odh-notebook-controller/kf-notebook-controller" @@ -17,6 +18,7 @@ declare -A COMPONENT_MANIFESTS=( ["model-mesh"]="opendatahub-io:modelmesh-serving:release-0.11.0:config:model-mesh" ["odh-model-controller"]="opendatahub-io:odh-model-controller:release-0.11.0:config:odh-model-controller" ["kserve"]="opendatahub-io:kserve:release-v0.11.0:config:kserve" + ["modelregistry"]="opendatahub-io:model-registry-operator:main:config:model-registry-operator" ) # Allow overwriting repo using flags component=repo @@ -43,10 +45,12 @@ if [ "$#" -ge 1 ]; then done fi -# R.I.P, odh-manifests +TMP_DIR=$(mktemp -d -t "odh-manifests.XXXXXXXXXX") +trap '{ rm -rf -- "$TMP_DIR"; }' EXIT + for key in "${!COMPONENT_MANIFESTS[@]}"; do - echo "Cloning repo ${key}: ${COMPONENT_MANIFESTS[$key]}" + echo -e "\033[32mCloning repo \033[33m${key}\033[32m:\033[0m ${COMPONENT_MANIFESTS[$key]}" IFS=':' read -r -a repo_info <<< "${COMPONENT_MANIFESTS[$key]}" repo_org="${repo_info[0]}" @@ -55,10 +59,12 @@ for key in "${!COMPONENT_MANIFESTS[@]}"; do source_path="${repo_info[3]}" target_path="${repo_info[4]}" - repo_url="${GITHUB_URL}/${repo_org}/${repo_name}.git" - rm -rf ./.${repo_name} - git clone --depth 1 --branch ${repo_branch} ${repo_url} ./.${repo_name} + repo_url="${GITHUB_URL}/${repo_org}/${repo_name}" + repo_dir=${TMP_DIR}/${key} + mkdir -p ${repo_dir} + git clone -q --depth 1 --branch ${repo_branch} ${repo_url} ${repo_dir} + mkdir -p ./odh-manifests/${target_path} - cp -rf ./.${repo_name}/${source_path}/* ./odh-manifests/${target_path} - rm -rf ./.${repo_name} + cp -rf ${repo_dir}/${source_path}/* ./odh-manifests/${target_path} + done diff --git a/go.mod b/go.mod index 651e4a73d79..cc31b21f696 100644 --- a/go.mod +++ b/go.mod @@ -12,18 +12,18 @@ require ( github.com/openshift/addon-operator/apis v0.0.0-20230919043633-820afed15881 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/custom-resource-status v1.1.2 - github.com/operator-framework/api v0.17.6 - github.com/operator-framework/operator-lifecycle-manager v0.18.3 + github.com/operator-framework/api v0.18.0 + github.com/operator-framework/operator-lifecycle-manager v0.26.0 github.com/pkg/errors v0.9.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 github.com/stretchr/testify v1.8.3 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.28.2 + k8s.io/api v0.28.3 k8s.io/apiextensions-apiserver v0.28.2 - k8s.io/apimachinery v0.28.2 + k8s.io/apimachinery v0.28.3 k8s.io/client-go v11.0.0+incompatible - k8s.io/kube-aggregator v0.23.0-alpha.1 + k8s.io/kube-aggregator v0.28.3 sigs.k8s.io/controller-runtime v0.16.1 sigs.k8s.io/kustomize/api v0.13.4 sigs.k8s.io/kustomize/kyaml v0.14.2 @@ -34,7 +34,7 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/emicklei/go-restful/v3 v3.10.2 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -53,7 +53,7 @@ require ( github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -71,7 +71,7 @@ require ( github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring v0.61.1-rhobs1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sergi/go-diff v1.2.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect @@ -90,7 +90,7 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/component-base v0.28.2 // indirect + k8s.io/component-base v0.28.3 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect @@ -100,6 +100,7 @@ require ( ) replace ( + github.com/go-yaml/yaml => github.com/go-yaml/yaml v2.2.8+incompatible github.com/tektoncd/pipeline => github.com/tektoncd/pipeline v0.12.0 go.uber.org/multierr => go.uber.org/multierr v1.6.0 k8s.io/api => k8s.io/api v0.26.0 diff --git a/go.sum b/go.sum index 233f40ae6f0..e571c958055 100644 --- a/go.sum +++ b/go.sum @@ -1,442 +1,97 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= -github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig/v3 v3.2.0/go.mod h1:tWhwTbUTndesPNeF0C900vKoq283u6zp4APT9vaF3SI= -github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= -github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/bugsnag-go v1.5.3/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/containerd v1.2.7/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= -github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= -github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= -github.com/cznic/internal v0.0.0-20180608152220-f44710a21d00/go.mod h1:olo7eAdKwJdXxb55TKGLiJ6xt1H0/tiiRCWKVLmtjY4= -github.com/cznic/lldb v1.1.0/go.mod h1:FIZVUmYUVhPwRiPzL8nD/mpFcJ/G7SSXjjXYG4uRI3A= -github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/ql v1.2.0/go.mod h1:FbpzhyZrqr0PVlK6ury+PoW3T0ODUV22OeWIxcaOrSE= -github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= -github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/cznic/zappy v0.0.0-20160723133515-2533cb5b45cc/go.mod h1:Y1SNZ4dRUOKXshKUbwUapqNncRrho4mkjQebgEHZLj8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= -github.com/deislabs/oras v0.8.1/go.mod h1:Mx0rMSbBNaNfY9hjpccEnxkOqJL6KGjtxNHPLC4G4As= -github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= -github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c1yc= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v0.0.0-20200130152716-5d0cf8839492/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v0.0.0-20191216044856-a8371794149d/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= -github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.7.3-0.20190103212154-2b7e084dc98b/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v0.7.3-0.20190817195342-4760db040282/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= +github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0yfBxPvZrHkprdPPTTS3N5rwmLE8T22KBXlw= -github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/fsouza/fake-gcs-server v1.7.0/go.mod h1:5XIRs4YvwNbNoz+1JF8j6KLAyDh7RHGAyAK3EP2EsNk= -github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= -github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= -github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= -github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= -github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= -github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/goccy/go-yaml v1.8.1/go.mod h1:wS4gNoLalDSJxo/SpngzPQ2BN4uuZVLCmbM4S3vd4+Y= -github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= -github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-migrate/migrate/v4 v4.6.2/go.mod h1:JYi6reN3+Z734VZ0akNuyOJNcrg45ZL7LDBMW3WGJL0= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -446,21 +101,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= -github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= @@ -469,289 +112,78 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-health-probe v0.3.2/go.mod h1:izVOQ4RWbjUR6lm4nn+VLJyQ+FyaiGmprEYgI04Gs7U= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/irifrance/gini v1.0.1/go.mod h1:swH5OTtiG/X/YrU06r288qZwq6I1agpbuXQOB55xqGU= -github.com/itchyny/astgen-go v0.0.0-20200519013840-cf3ea398f645/go.mod h1:296z3W7Xsrp2mlIY88ruDKscuvrkL6zXCNRtaYVshzw= -github.com/itchyny/go-flags v1.5.0/go.mod h1:lenkYuCobuxLBAd/HGFE4LRoW8D3B6iXRQfWYJ+MNbA= -github.com/itchyny/gojq v0.11.0/go.mod h1:my6D2qN2Sm6qa+/5GsPDUZlCWGR+U8Qsa9he76sudv0= -github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= -github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= -github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869/go.mod h1:cJ6Cj7dQo+O6GJNiMx+Pa94qKj+TG8ONdKHgMNIyyag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kshvakov/clickhouse v1.3.5/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= -github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/strftime v1.0.1/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mikefarah/yaml/v2 v2.4.0/go.mod h1:ahVqZF4n1W4NqwvVnZzC4es67xsW9uR/RRf2RRxieJU= -github.com/mikefarah/yq/v2 v2.4.1/go.mod h1:i8SYf1XdgUvY2OFwSqGAtWOOgimD2McJ6iutoxRm4k0= -github.com/mikefarah/yq/v3 v3.0.0-20201202084205-8846255d1c37/go.mod h1:dYWq+UWoFCDY1TndvFUQuhBbIYmZpjreC8adEAx93zE= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= -github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= -github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= @@ -759,18 +191,10 @@ github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47 github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= -github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= @@ -778,210 +202,56 @@ github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeR github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opendatahub-io/opendatahub-operator v1.7.0 h1:Pn76VWCUHeqD5E0L94t5FtZ2OU6ZaWk/IwbEqXGQ4Gs= github.com/opendatahub-io/opendatahub-operator v1.7.0/go.mod h1:XsgkXbGjJoVeZOkmRztemhY5ppX7puzuqDw9oUdzNZk= github.com/openshift/addon-operator/apis v0.0.0-20230919043633-820afed15881 h1:d0hmj9Is2sCLkNYWtBicZV0Ft8+Os+w+RUkFRjie0VI= github.com/openshift/addon-operator/apis v0.0.0-20230919043633-820afed15881/go.mod h1:2hsK4sYLKcjVJ8SziFrzr/c/Tmp5zBDy8aYvrFaRm2o= -github.com/openshift/api v0.0.0-20200326152221-912866ddb162/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE= -github.com/openshift/api v0.0.0-20200331152225-585af27e34fd/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE= github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 h1:e3zIxk67/kiABxGFfFVECqJ4FcQRG5DPF8lgDV9f+MM= github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= -github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc= -github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0/go.mod h1:uUQ4LClRO+fg5MF/P6QxjMCb1C9f7Oh4RKepftDnEJE= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/operator-framework/api v0.3.7-0.20200602203552-431198de9fc2/go.mod h1:Xbje9x0SHmh0nihE21kpesB38vk3cyxnE6JdDS8Jo1Q= -github.com/operator-framework/api v0.9.2/go.mod h1:33GnP7NpUNEl5o8ri/sX3ns+jJv7jW/36gOmfU8emX0= -github.com/operator-framework/api v0.17.6 h1:E6+vlvYUKafvoXYtCuHlDZrXX4vl8AT+r93OxNlzjpU= -github.com/operator-framework/api v0.17.6/go.mod h1:l/cuwtPxkVUY7fzYgdust2m9tlmb8I4pOvbsUufRb24= -github.com/operator-framework/operator-lifecycle-manager v0.18.3 h1:O0GGNPM7OfpsBAU/j6wLZTw2bs/t7TFfIJNfe4Zkxlk= -github.com/operator-framework/operator-lifecycle-manager v0.18.3/go.mod h1:OA+0OqibpOKs7EOwvbby34xxNtgfoh61vGuKnbA5WCw= -github.com/operator-framework/operator-registry v1.13.6/go.mod h1:YhnIzOVjRU2ZwZtzt+fjcjW8ujJaSFynBEu7QVKaSdU= -github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pbnjay/strptime v0.0.0-20140226051138-5c05b0d668c9/go.mod h1:6Hr+C/olSdkdL3z68MlyXWzwhvwmwN7KuUFXGb3PoOk= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/operator-framework/api v0.18.0 h1:6EdSNeAjin4LRu2YQnQWMJMc6HXS0AQDG+CfaEvFrAo= +github.com/operator-framework/api v0.18.0/go.mod h1:SCCslqke6AVOJ5JM+NqNE1CHuAgJLScsL66pnPaSMXs= +github.com/operator-framework/operator-lifecycle-manager v0.26.0 h1:16vEJZ5gzMXlcvNlJBnLsEfa+8h5+mvp7/xHwj+u230= +github.com/operator-framework/operator-lifecycle-manager v0.26.0/go.mod h1:uDY8iANE0neSPEYNsT/AE3fpuKS9OhgcKqKEBMhGO6A= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 h1:yl9ceUSUBo9woQIO+8eoWpcxZkdZgm89g+rVvu37TUw= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0/go.mod h1:9Uuu3pEU2jB8PwuqkHvegQ0HV/BlZRJUyfTYAqfdVF8= -github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring v0.61.1-rhobs1 h1:sI4OJX9/XkSd8O6/sY4cxJPiuwM1RHv3qygIbDpBoAY= github.com/rhobs/obo-prometheus-operator/pkg/apis/monitoring v0.61.1-rhobs1/go.mod h1:u8ctCYj9Nq8gkMLfNLxHoslu8SEGrqXP2gFiMUNsn9g= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -990,194 +260,43 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tebeka/strftime v0.1.3/go.mod h1:7wJm3dZlpr4l/oVK0t1HYIc4rMzQ2XJlOMIUJUJH6XQ= -github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/gorelic v0.0.7/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= -go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= -go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= -go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= @@ -1188,209 +307,70 @@ golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190102155601-82a175fd1598/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190426135247-a129542de9ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1399,24 +379,18 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -1424,88 +398,18 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190425222832-ad9eeb80039a/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200115044656-831fdb1e1868/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= @@ -1520,156 +424,25 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200701001935-0939c5918c31/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v0.0.0-20200709232328-d8193ee9cc3e/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1687,186 +460,75 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.30.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= -gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= -gopkg.in/imdario/mergo.v0 v0.3.7/go.mod h1:9qPP6AGrlC1G2PTNXko614FwGZvorN7MiBU0Eppok+U= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -helm.sh/helm/v3 v3.1.0-rc.1.0.20201215141456-e71d38b414eb/go.mod h1:Y5K3Kpp4CgPLcW6KgR8FmW93jrdo0HPhA7/MPOSkMbw= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= -k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= -k8s.io/apiextensions-apiserver v0.20.0/go.mod h1:ZH+C33L2Bh1LY1+HphoRmN1IQVLTShVcTojivK3N9xg= -k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= -k8s.io/apiextensions-apiserver v0.20.6/go.mod h1:qO8YMqeMmZH+lV21LUNzV41vfpoE9QVAJRA+MNqj0mo= -k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU= k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg= -k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.23.0-alpha.1/go.mod h1:rm7GDMZSKVlauLBWTPBAhG0qULDXZtGEoit5cRQd/+Y= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= -k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.23.0-alpha.1/go.mod h1:6BMSifW1nLddaKBt7pYtIg837dzU5GYj2PuKMltARAk= -k8s.io/apiserver v0.26.1/go.mod h1:wr75z634Cv+sifswE9HlAo5FQ7UoUauIICRlOE+5dCg= -k8s.io/cli-runtime v0.18.0/go.mod h1:1eXfmBsIJosjn9LjEBUd2WVPoPAY9XGTqTFcPMIBsUQ= -k8s.io/cli-runtime v0.20.0/go.mod h1:C5tewU1SC1t09D7pmkk83FT4lMAw+bvMDuRxA7f0t2s= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= -k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.20.6/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU= -k8s.io/code-generator v0.23.0-alpha.1/go.mod h1:693dDHJwzTStZeGqGAgEbvL123abW2AyBJNAPoChgmw= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I= -k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= -k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= -k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.23.0-alpha.1/go.mod h1:95CInsMjswkg3eWTrDm1GESMGI2l6e8bRalAfYbTdC4= -k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= -k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E= -k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc= -k8s.io/component-helpers v0.20.0/go.mod h1:nx6NOtfSfGOxnSZsDJxpGbnsVuUA1UXpwDvZIrtigNk= -k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.26.1/go.mod h1:ReC1IEGuxgfN+PDCIpR6w8+XMmDE7uJhxcCwMZFdIYc= -k8s.io/kube-aggregator v0.20.4/go.mod h1:0ixQ9De7KXyHteXizS6nVtrnKqGa4kiuxl9rEBsNccw= -k8s.io/kube-aggregator v0.23.0-alpha.1 h1:rVvz3y4rJPYSJs/rNwX9qDYhxSP9i/Hyzd9FRz0mTUY= -k8s.io/kube-aggregator v0.23.0-alpha.1/go.mod h1:uVAgJcfmwbSLsZrG9RQZPyLMJqHyWRbPkxvLbkdOHDk= +k8s.io/kube-aggregator v0.28.3 h1:CVbj3+cpshSHR5dWPzLYx3sVpIDEPLlzMSxY/lAc9cM= +k8s.io/kube-aggregator v0.28.3/go.mod h1:5DyLevbRTcWnT1f9b+lB3BfbXC1w7gDa/OtB6kKInCw= k8s.io/kube-openapi v0.0.0-20230113010536-ace3e4a52ca1 h1:hoNFU8otQDVH5xJiqsJ2E86d8v87t2Uz1K2WuiNSNDY= k8s.io/kube-openapi v0.0.0-20230113010536-ace3e4a52ca1/go.mod h1:/BYxry62FuDzmI+i9B+X2pqfySRmSOW2ARmj5Zbqhj0= -k8s.io/kubectl v0.18.0/go.mod h1:LOkWx9Z5DXMEg5KtOjHhRiC1fqJPLyCr3KtQgEolCkU= -k8s.io/kubectl v0.20.0/go.mod h1:8x5GzQkgikz7M2eFGGuu6yOfrenwnw5g4RXOUgbjR1M= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/metrics v0.18.0/go.mod h1:8aYTW18koXqjLVKL7Ds05RPMX9ipJZI3mywYvBOxXd4= -k8s.io/metrics v0.20.0/go.mod h1:9yiRhfr8K8sjdj2EthQQE9WvpYDvsXIV3CjN4Ruq4Jw= -k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/letsencrypt v0.0.3/go.mod h1:buyQKZ6IXrRnB7TdkHP0RyEybLx18HHyOSoTyoOLqNY= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= -sigs.k8s.io/controller-tools v0.3.0/go.mod h1:enhtKGfxZD1GFEoMgP8Fdbu+uKQ/cq1/WGJhdVChfvI= -sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kind v0.11.1/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA= -sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.13.4 h1:E38Hfx0G9R9v7vRgKshviPotJQETG0S2gD3JdHLCAsI= sigs.k8s.io/kustomize/api v0.13.4/go.mod h1:Bkaavz5RKK6ZzP0zgPrB7QbpbBJKiHuD3BB0KujY7Ls= sigs.k8s.io/kustomize/kyaml v0.14.2 h1:9WSwztbzwGszG1bZTziQUmVMrJccnyrLb5ZMKpJGvXw= sigs.k8s.io/kustomize/kyaml v0.14.2/go.mod h1:AN1/IpawKilWD7V+YvQwRGUvuUOOWpjsHu6uHwonSF4= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/infrastructure/v1/serverless_types.go b/infrastructure/v1/serverless_types.go index 37155c1488e..c5022a85e91 100644 --- a/infrastructure/v1/serverless_types.go +++ b/infrastructure/v1/serverless_types.go @@ -7,8 +7,8 @@ import ( // ServingSpec specifies the configuration for the KNative Serving components and their // bindings with the Service Mesh. type ServingSpec struct { - // +kubebuilder:validation:Enum=Managed;Removed - // +kubebuilder:default=Removed + // +kubebuilder:validation:Enum=Managed;Unmanaged;Removed + // +kubebuilder:default=Managed ManagementState operatorv1.ManagementState `json:"managementState,omitempty"` // Name specifies the name of the KNativeServing resource that is going to be // created to instruct the KNative Operator to deploy KNative serving components. diff --git a/infrastructure/v1/servicemesh_types.go b/infrastructure/v1/servicemesh_types.go index db4af44d712..9c093d0c663 100644 --- a/infrastructure/v1/servicemesh_types.go +++ b/infrastructure/v1/servicemesh_types.go @@ -4,7 +4,7 @@ import operatorv1 "github.com/openshift/api/operator/v1" // ServiceMeshSpec configures Service Mesh. type ServiceMeshSpec struct { - // +kubebuilder:validation:Enum=Managed;Removed + // +kubebuilder:validation:Enum=Managed;Unmanaged;Removed // +kubebuilder:default=Removed ManagementState operatorv1.ManagementState `json:"managementState,omitempty"` // ControlPlane holds configuration of Service Mesh used by Opendatahub. diff --git a/main.go b/main.go index 030786c9a7a..eb2103d929a 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,8 @@ import ( kfdefv1 "github.com/opendatahub-io/opendatahub-operator/apis/kfdef.apps.kubeflow.org/v1" dsc "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1" dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" - dscontr "github.com/opendatahub-io/opendatahub-operator/v2/controllers/datasciencecluster" + featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1" + datascienceclustercontrollers "github.com/opendatahub-io/opendatahub-operator/v2/controllers/datasciencecluster" dscicontr "github.com/opendatahub-io/opendatahub-operator/v2/controllers/dscinitialization" "github.com/opendatahub-io/opendatahub-operator/v2/controllers/secretgenerator" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" @@ -60,11 +61,12 @@ var ( setupLog = ctrl.Log.WithName("setup") ) -func init() { +func init() { //nolint:gochecknoinits //+kubebuilder:scaffold:scheme utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(dsci.AddToScheme(scheme)) utilruntime.Must(dsc.AddToScheme(scheme)) + utilruntime.Must(featurev1.AddToScheme(scheme)) utilruntime.Must(netv1.AddToScheme(scheme)) utilruntime.Must(addonv1alpha1.AddToScheme(scheme)) utilruntime.Must(authv1.AddToScheme(scheme)) @@ -143,12 +145,12 @@ func main() { //nolint:funlen os.Exit(1) } - if err = (&dscontr.DataScienceClusterReconciler{ + if err = (&datascienceclustercontrollers.DataScienceClusterReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), RestConfig: mgr.GetConfig(), Log: ctrl.Log.WithName("controllers").WithName("DataScienceCluster"), - DataScienceCluster: &dscontr.DataScienceClusterConfig{ + DataScienceCluster: &datascienceclustercontrollers.DataScienceClusterConfig{ DSCISpec: &dsci.DSCInitializationSpec{ ApplicationsNamespace: dscApplicationsNamespace, }, @@ -182,7 +184,7 @@ func main() { //nolint:funlen // Get operator platform platform, err := deploy.GetPlatform(setupClient) if err != nil { - setupLog.Error(err, "error getting client for setup") + setupLog.Error(err, "error getting platform") os.Exit(1) } @@ -195,7 +197,7 @@ func main() { //nolint:funlen } // Apply update from legacy operator - if err = upgrade.UpdateFromLegacyVersion(setupClient, platform); err != nil { + if err = upgrade.UpdateFromLegacyVersion(setupClient, platform, dscApplicationsNamespace, dscMonitoringNamespace); err != nil { setupLog.Error(err, "unable to update from legacy operator version") } diff --git a/pkg/cluster/operations.go b/pkg/cluster/operations.go index d9d67716803..3f8a1ca1ae7 100644 --- a/pkg/cluster/operations.go +++ b/pkg/cluster/operations.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" authv1 "k8s.io/api/rbac/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" @@ -21,9 +20,8 @@ const ( // being used by different components. func UpdatePodSecurityRolebinding(cli client.Client, namespace string, serviceAccountsList ...string) error { foundRoleBinding := &authv1.RoleBinding{} - err := cli.Get(context.TODO(), client.ObjectKey{Name: namespace, Namespace: namespace}, foundRoleBinding) - if err != nil { - return err + if err := cli.Get(context.TODO(), client.ObjectKey{Name: namespace, Namespace: namespace}, foundRoleBinding); err != nil { + return fmt.Errorf("error to get rolebinding %s from namespace %s: %w", namespace, namespace, err) } for _, sa := range serviceAccountsList { @@ -37,7 +35,11 @@ func UpdatePodSecurityRolebinding(cli client.Client, namespace string, serviceAc } } - return cli.Update(context.TODO(), foundRoleBinding) + if err := cli.Update(context.TODO(), foundRoleBinding); err != nil { + return fmt.Errorf("error update rolebinding %s with serviceaccount: %w", namespace, err) + } + + return nil } // Internal function used by UpdatePodSecurityRolebinding() @@ -117,7 +119,7 @@ func WithLabels(labels ...string) MetaOptions { return func(obj metav1.Object) error { labelsMap, err := extractKeyValues(labels) if err != nil { - return errors.Wrap(err, "unable to set labels") + return fmt.Errorf("failed unable to set labels: %w", err) } obj.SetLabels(labelsMap) diff --git a/pkg/deploy/deploy.go b/pkg/deploy/deploy.go index 672adb50c8a..23aa4d79aba 100644 --- a/pkg/deploy/deploy.go +++ b/pkg/deploy/deploy.go @@ -23,6 +23,7 @@ import ( "compress/gzip" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -58,9 +59,13 @@ const ( func DownloadManifests(componentName string, manifestConfig components.ManifestsConfig) error { // Get the component repo from the given url // e.g https://github.com/example/tarball/master - resp, err := http.Get(manifestConfig.URI) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, manifestConfig.URI, nil) if err != nil { - return fmt.Errorf("error downloading manifests: %v", err) + return err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("error downloading manifests: %w", err) } defer resp.Body.Close() @@ -71,7 +76,7 @@ func DownloadManifests(componentName string, manifestConfig components.Manifests // Create a new gzip reader gzipReader, err := gzip.NewReader(resp.Body) if err != nil { - return fmt.Errorf("error creating gzip reader: %v", err) + return fmt.Errorf("error creating gzip reader: %w", err) } defer gzipReader.Close() @@ -82,13 +87,13 @@ func DownloadManifests(componentName string, manifestConfig components.Manifests mode := os.ModePerm err = os.MkdirAll(DefaultManifestPath, mode) if err != nil { - return fmt.Errorf("error creating manifests directory : %v", err) + return fmt.Errorf("error creating manifests directory : %w", err) } // Extract the contents of the TAR archive to the current directory for { header, err := tarReader.Next() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } if err != nil { @@ -108,7 +113,7 @@ func DownloadManifests(componentName string, manifestConfig components.Manifests if header.Typeflag == tar.TypeDir { err = os.MkdirAll(DefaultManifestPath+"/"+componentName+"/"+componentFileRelativePathFound, mode) if err != nil { - return fmt.Errorf("error creating directory:%v", err) + return fmt.Errorf("error creating directory:%w", err) } continue @@ -119,9 +124,15 @@ func DownloadManifests(componentName string, manifestConfig components.Manifests if err != nil { fmt.Println("Error creating file:", err) } - _, err = io.Copy(file, tarReader) - if err != nil { - fmt.Println("Error downloading file contents:", err) + for { + _, err := io.CopyN(file, tarReader, 1024) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + fmt.Println("Error downloading file contents:", err) + return err + } } file.Close() @@ -133,7 +144,7 @@ func DownloadManifests(componentName string, manifestConfig components.Manifests return err } -func DeployManifestsFromPath(cli client.Client, owner metav1.Object, manifestPath string, namespace string, componentName string, componentEnabled bool) error { +func DeployManifestsFromPath(cli client.Client, owner metav1.Object, manifestPath string, namespace string, componentName string, componentEnabled bool) error { //nolint:golint,revive,lll // Render the Kustomize manifests k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) fs := filesys.MakeFsOnDisk() @@ -180,7 +191,7 @@ func DeployManifestsFromPath(cli client.Client, owner metav1.Object, manifestPat } func getResources(resMap resmap.ResMap) ([]*unstructured.Unstructured, error) { - var resources []*unstructured.Unstructured + resources := make([]*unstructured.Unstructured, 0, resMap.Size()) for _, res := range resMap.Resources() { u := &unstructured.Unstructured{} err := yaml.Unmarshal([]byte(res.MustYaml()), u) @@ -208,9 +219,8 @@ func manageResource(ctx context.Context, cli client.Client, obj *unstructured.Un // Resource exists but component is disabled if !enabled { - // Return nil for any errors getting the resource, since the component itself is disabled if err != nil { - return nil + return nil //nolint:nilerr // Return nil for any errors getting the resource, since the component itself is disabled } // Check for shared resources before deletion @@ -236,7 +246,10 @@ func manageResource(ctx context.Context, cli client.Client, obj *unstructured.Un } } - if obj.GetOwnerReferences() == nil { + existingOwnerReferences := obj.GetOwnerReferences() + selector := "app.opendatahub.io/" + componentName + // only removed the resource with our label applied, not the same name resource maually created by user + if existingOwnerReferences == nil && resourceLabels[selector] == "true" { return cli.Delete(ctx, found) } @@ -395,9 +408,8 @@ func SubscriptionExists(cli client.Client, namespace string, name string) (bool, if err := cli.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: name}, sub); err != nil { if apierrs.IsNotFound(err) { return false, nil - } else { - return false, err } + return false, err } return true, nil diff --git a/pkg/deploy/setup.go b/pkg/deploy/setup.go index dfcb4809164..206ef9eab49 100644 --- a/pkg/deploy/setup.go +++ b/pkg/deploy/setup.go @@ -33,7 +33,7 @@ func isSelfManaged(cli client.Client) (Platform, error) { err := cli.List(context.TODO(), clusterCsvs) if err != nil { return "", err - } else { + } else { //nolint:golint,revive // Readability on else for _, csv := range clusterCsvs.Items { if strings.Contains(csv.Spec.DisplayName, string(OpenDataHub)) { return OpenDataHub, nil @@ -58,26 +58,24 @@ func isManagedRHODS(cli client.Client) (Platform, error) { } return "", err - } else { - expectedCatlogSource := &ofapi.CatalogSourceList{} - err := cli.List(context.TODO(), expectedCatlogSource) - if err != nil { - if apierrs.IsNotFound(err) { - return Unknown, nil - } else { - return Unknown, err - } + } + expectedCatlogSource := &ofapi.CatalogSourceList{} + err = cli.List(context.TODO(), expectedCatlogSource) + if err != nil { + if apierrs.IsNotFound(err) { + return Unknown, nil } - if len(expectedCatlogSource.Items) > 0 { - for _, cs := range expectedCatlogSource.Items { - if cs.Name == string(ManagedRhods) { - return ManagedRhods, nil - } + return Unknown, err + } + if len(expectedCatlogSource.Items) > 0 { + for _, cs := range expectedCatlogSource.Items { + if cs.Name == string(ManagedRhods) { + return ManagedRhods, nil } } - - return "", nil } + + return "", nil } func GetPlatform(cli client.Client) (Platform, error) { diff --git a/pkg/feature/builder.go b/pkg/feature/builder.go index 55b17256048..e019ecf555c 100644 --- a/pkg/feature/builder.go +++ b/pkg/feature/builder.go @@ -1,6 +1,9 @@ package feature import ( + "io/fs" + + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/client-go/dynamic" @@ -10,32 +13,48 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" - v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1" infrav1 "github.com/opendatahub-io/opendatahub-operator/v2/infrastructure/v1" ) type partialBuilder func(f *Feature) error type featureBuilder struct { - name string - builders []partialBuilder + name string + config *rest.Config + builders []partialBuilder + featuresHanlder *FeaturesHandler + fsys fs.FS +} + +func CreateFeature(name string) *usingFeaturesHandler { //nolint:golint,revive //No need to export featureBuilder. + return &usingFeaturesHandler{ + name: name, + } } -func CreateFeature(name string) *featureBuilder { - return &featureBuilder{name: name} +type usingFeaturesHandler struct { + name string } -func (fb *featureBuilder) For(spec *v1.DSCInitializationSpec) *featureBuilder { +func (u *usingFeaturesHandler) For(featuresHandler *FeaturesHandler) *featureBuilder { createSpec := func(f *Feature) error { f.Spec = &Spec{ - ServiceMeshSpec: &spec.ServiceMesh, + ServiceMeshSpec: &featuresHandler.DSCInitializationSpec.ServiceMesh, Serving: &infrav1.ServingSpec{}, - AppNamespace: spec.ApplicationsNamespace, + Source: &featuresHandler.source, + AppNamespace: featuresHandler.DSCInitializationSpec.ApplicationsNamespace, } return nil } + fb := &featureBuilder{ + name: u.name, + featuresHanlder: featuresHandler, + fsys: embeddedFiles, + } + // Ensures creation of .Spec object is always invoked first fb.builders = append([]partialBuilder{createSpec}, fb.builders...) @@ -43,8 +62,7 @@ func (fb *featureBuilder) For(spec *v1.DSCInitializationSpec) *featureBuilder { } func (fb *featureBuilder) UsingConfig(config *rest.Config) *featureBuilder { - fb.builders = append(fb.builders, createClients(config)) - + fb.config = config return fb } @@ -66,11 +84,11 @@ func createClients(config *rest.Config) partialBuilder { return errors.WithStack(err) } - if err := apiextv1.AddToScheme(f.Client.Scheme()); err != nil { - return err - } + var multiErr *multierror.Error + s := f.Client.Scheme() + multiErr = multierror.Append(multiErr, featurev1.AddToScheme(s), apiextv1.AddToScheme(s)) - return nil + return multiErr.ErrorOrNil() } } @@ -80,7 +98,7 @@ func (fb *featureBuilder) Manifests(paths ...string) *featureBuilder { var manifests []manifest for _, path := range paths { - manifests, err = loadManifestsFrom(path) + manifests, err = loadManifestsFrom(fb.fsys, path) if err != nil { return errors.WithStack(err) } @@ -144,47 +162,56 @@ func (fb *featureBuilder) WithResources(resources ...Action) *featureBuilder { return fb } -func (fb *featureBuilder) Load() (*Feature, error) { - feature := &Feature{ - Name: fb.name, - Enabled: true, +func (fb *featureBuilder) Load() error { + feature := newFeature(fb.name) + + // UsingConfig builder wasn't called while constructing this feature. + // Get default settings and create needed clients. + if fb.config == nil { + if err := fb.withDefaultClient(); err != nil { + return err + } + } + + if err := createClients(fb.config)(feature); err != nil { + return err } for i := range fb.builders { if err := fb.builders[i](feature); err != nil { - return nil, err + return err } } - // UsingConfig builder wasn't called while constructing this feature. - // Get default settings and create needed clients. - if feature.Client == nil { - restCfg, err := config.GetConfig() - if errors.Is(err, rest.ErrNotInCluster) { - // rollback to local kubeconfig - this can be helpful when running the process locally i.e. while debugging - kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - &clientcmd.ClientConfigLoadingRules{ExplicitPath: clientcmd.RecommendedHomeFile}, - &clientcmd.ConfigOverrides{}, - ) - - restCfg, err = kubeconfig.ClientConfig() - if err != nil { - return nil, err - } - } else if err != nil { - return nil, err - } + fb.featuresHanlder.features = append(fb.featuresHanlder.features, feature) - if err := createClients(restCfg)(feature); err != nil { - return nil, err - } - } + return nil +} - if feature.Enabled { - if err := feature.createResourceTracker(); err != nil { - return feature, err +func (fb *featureBuilder) withDefaultClient() error { + restCfg, err := config.GetConfig() + if errors.Is(err, rest.ErrNotInCluster) { + // rollback to local kubeconfig - this can be helpful when running the process locally i.e. while debugging + kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: clientcmd.RecommendedHomeFile}, + &clientcmd.ConfigOverrides{}, + ) + + restCfg, err = kubeconfig.ClientConfig() + if err != nil { + return err } + } else if err != nil { + return err } - return feature, nil + fb.config = restCfg + return nil +} + +// ManifestSource sets the root file system (fsys) from which manifest paths are loaded +// If ManifestSource is not called in the builder chain, the default source will be the embeddedFiles. +func (fb *featureBuilder) ManifestSource(fsys fs.FS) *featureBuilder { + fb.fsys = fsys + return fb } diff --git a/pkg/feature/cert.go b/pkg/feature/cert.go index 020b2dea1b6..64b6d05c17a 100644 --- a/pkg/feature/cert.go +++ b/pkg/feature/cert.go @@ -3,13 +3,12 @@ package feature import ( "bytes" "context" - cryptorand "crypto/rand" + "crypto/rand" "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "encoding/pem" "math/big" - "math/rand" "net" "strings" "time" @@ -22,8 +21,6 @@ import ( infrav1 "github.com/opendatahub-io/opendatahub-operator/v2/infrastructure/v1" ) -var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) - func (f *Feature) CreateSelfSignedCertificate(secretName string, certificateType infrav1.CertType, domain, namespace string) error { if certificateType != infrav1.SelfSigned { return nil @@ -33,7 +30,7 @@ func (f *Feature) CreateSelfSignedCertificate(secretName string, certificateType Name: secretName, Namespace: namespace, OwnerReferences: []metav1.OwnerReference{ - f.OwnerReference(), + f.AsOwnerReference(), }, } @@ -72,14 +69,19 @@ func GenerateSelfSignedCertificateAsSecret(addr string, objectMeta metav1.Object } func generateCertificate(addr string) ([]byte, []byte, error) { - key, err := rsa.GenerateKey(cryptorand.Reader, 2048) + key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { return nil, nil, errors.WithStack(err) } + seededRand, crypterr := rand.Int(rand.Reader, big.NewInt(time.Now().UnixNano())) + if err != nil { + return nil, nil, errors.WithStack(crypterr) + } + now := time.Now() tmpl := x509.Certificate{ - SerialNumber: new(big.Int).SetInt64(seededRand.Int63()), + SerialNumber: seededRand, Subject: pkix.Name{ CommonName: addr, Organization: []string{"opendatahub-self-signed"}, @@ -103,7 +105,7 @@ func generateCertificate(addr string) ([]byte, []byte, error) { tmpl.DNSNames = append(tmpl.DNSNames, "localhost") - certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key) + certDERBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, key.Public(), key) if err != nil { return nil, nil, errors.WithStack(err) } diff --git a/pkg/feature/conditions.go b/pkg/feature/conditions.go index 86b662f0d3e..8ee161f96c4 100644 --- a/pkg/feature/conditions.go +++ b/pkg/feature/conditions.go @@ -24,11 +24,11 @@ func EnsureCRDIsInstalled(name string) Action { } func WaitForPodsToBeReady(namespace string) Action { - return func(feature *Feature) error { - log.Info("waiting for pods to become ready", "feature", feature.Name, "namespace", namespace, "duration (s)", duration.Seconds()) + return func(f *Feature) error { + f.Log.Info("waiting for pods to become ready", "namespace", namespace, "duration (s)", duration.Seconds()) return wait.PollUntilContextTimeout(context.TODO(), interval, duration, false, func(ctx context.Context) (bool, error) { - podList, err := feature.Clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) + podList, err := f.Clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) if err != nil { return false, err } @@ -36,6 +36,10 @@ func WaitForPodsToBeReady(namespace string) Action { readyPods := 0 totalPods := len(podList.Items) + if totalPods == 0 { // We want to wait for "something", so make sure we have "something" before we claim success. + return false, nil + } + for _, pod := range podList.Items { podReady := true // Consider a "PodSucceeded" as ready, since these will never will @@ -59,7 +63,7 @@ func WaitForPodsToBeReady(namespace string) Action { done := readyPods == totalPods if done { - log.Info("done waiting for pods to become ready", "feature", feature.Name, "namespace", namespace) + f.Log.Info("done waiting for pods to become ready", "namespace", namespace) } return done, nil @@ -68,19 +72,19 @@ func WaitForPodsToBeReady(namespace string) Action { } func WaitForResourceToBeCreated(namespace string, gvr schema.GroupVersionResource) Action { - return func(feature *Feature) error { - log.Info("waiting for resource to be created", "namespace", namespace, "resource", gvr) + return func(f *Feature) error { + f.Log.Info("waiting for resource to be created", "namespace", namespace, "resource", gvr) return wait.PollUntilContextTimeout(context.TODO(), interval, duration, false, func(ctx context.Context) (bool, error) { - resources, err := feature.DynamicClient.Resource(gvr).Namespace(namespace).List(context.TODO(), metav1.ListOptions{Limit: 1}) + resources, err := f.DynamicClient.Resource(gvr).Namespace(namespace).List(context.TODO(), metav1.ListOptions{Limit: 1}) if err != nil { - log.Error(err, "failed waiting for resource", "namespace", namespace, "resource", gvr) + f.Log.Error(err, "failed waiting for resource", "namespace", namespace, "resource", gvr) return false, err } if len(resources.Items) > 0 { - log.Info("resource created", "namespace", namespace, "resource", gvr) + f.Log.Info("resource created", "namespace", namespace, "resource", gvr) return true, nil } diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 397fda734a3..5ccb5d6f3a5 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -2,26 +2,23 @@ package feature import ( "context" + "fmt" + "github.com/go-logr/logr" "github.com/hashicorp/go-multierror" + conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/common" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" ) -var log = ctrlLog.Log.WithName("features") - type Feature struct { Name string Spec *Spec @@ -31,80 +28,108 @@ type Feature struct { DynamicClient dynamic.Interface Client client.Client - manifests []manifest + manifests []manifest + cleanups []Action resources []Action preconditions []Action postconditions []Action loaders []Action + + Log logr.Logger +} + +func newFeature(name string) *Feature { + return &Feature{ + Name: name, + Enabled: true, + Log: ctrlLog.Log.WithName("features").WithValues("feature", name), + } } // Action is a func type which can be used for different purposes while having access to Feature struct. type Action func(feature *Feature) error -func (f *Feature) Apply() error { +//nolint:nonamedreturns // Reason: we use the named return to handle errors in a unified fashion through deferred function. +func (f *Feature) Apply() (err error) { if !f.Enabled { - log.Info("feature is disabled, skipping.", "feature", f.Name) - return nil } + if trackerErr := f.createFeatureTracker(); err != nil { + return trackerErr + } + // Verify all precondition and collect errors var multiErr *multierror.Error + phase := featurev1.FeatureCreated + f.updateFeatureTrackerStatus(conditionsv1.ConditionProgressing, "True", phase, fmt.Sprintf("Applying feature %s", f.Name)) + defer func() { + if err != nil { + f.updateFeatureTrackerStatus(conditionsv1.ConditionDegraded, "True", phase, err.Error()) + } else { + f.updateFeatureTrackerStatus(conditionsv1.ConditionAvailable, "True", phase, fmt.Sprintf("Feature %s applied successfully", f.Name)) + } + }() + + phase = featurev1.PreConditions for _, precondition := range f.preconditions { multiErr = multierror.Append(multiErr, precondition(f)) } - if multiErr.ErrorOrNil() != nil { - return multiErr.ErrorOrNil() + if preconditionsErr := multiErr.ErrorOrNil(); preconditionsErr != nil { + return preconditionsErr } - // Load necessary data + phase = featurev1.LoadTemplateData for _, loader := range f.loaders { multiErr = multierror.Append(multiErr, loader(f)) } - if multiErr.ErrorOrNil() != nil { - return multiErr.ErrorOrNil() + + if dataLoadErr := multiErr.ErrorOrNil(); dataLoadErr != nil { + return dataLoadErr } - // create or update resources + phase = featurev1.ResourceCreation for _, resource := range f.resources { if err := resource(f); err != nil { - return err + return errors.WithStack(err) } } - // Process and apply manifests - for _, m := range f.manifests { - if err := m.processTemplate(f.Spec); err != nil { + phase = featurev1.ProcessTemplates + for i := range f.manifests { + if err := f.manifests[i].process(f.Spec); err != nil { return errors.WithStack(err) } - - log.Info("converted template to manifest", "feature", f.Name, "path", m.targetPath()) } + phase = featurev1.ApplyManifests if err := f.applyManifests(); err != nil { - return err + return errors.WithStack(err) } + phase = featurev1.PostConditions for _, postcondition := range f.postconditions { multiErr = multierror.Append(multiErr, postcondition(f)) } - if multiErr.ErrorOrNil() != nil { return multiErr.ErrorOrNil() } + phase = featurev1.FeatureCreated return nil } func (f *Feature) Cleanup() error { if !f.Enabled { - log.Info("feature is disabled, skipping.", "feature", f.Name) - return nil } + // Ensure associated FeatureTracker instance has been removed as last one + // in the chain of cleanups. + f.addCleanup(removeFeatureTracker) + var cleanupErrors *multierror.Error for _, cleanupFunc := range f.cleanups { cleanupErrors = multierror.Append(cleanupErrors, cleanupFunc(f)) @@ -128,7 +153,7 @@ func (f *Feature) CreateConfigMap(cfgMapName string, data map[string]string) err Name: cfgMapName, Namespace: f.Spec.AppNamespace, OwnerReferences: []metav1.OwnerReference{ - f.OwnerReference(), + f.AsOwnerReference(), }, }, Data: data, @@ -157,28 +182,28 @@ func (f *Feature) addCleanup(cleanupFuncs ...Action) { f.cleanups = append(f.cleanups, cleanupFuncs...) } -type apply func(filename string) error +type apply func(data string) error func (f *Feature) apply(m manifest) error { var applier apply targetPath := m.targetPath() if m.patch { - applier = func(filename string) error { - log.Info("patching using manifest", "feature", f.Name, "name", m.name, "path", targetPath) + applier = func(data string) error { + f.Log.Info("patching using manifest", "feature", f.Name, "name", m.name, "path", targetPath) - return f.patchResourceFromFile(filename) + return f.patchResources(data) } } else { - applier = func(filename string) error { - log.Info("applying manifest", "feature", f.Name, "name", m.name, "path", targetPath) + applier = func(data string) error { + f.Log.Info("applying manifest", "feature", f.Name, "name", m.name, "path", targetPath) - return f.createResourceFromFile(filename) + return f.createResources(data) } } - if err := applier(targetPath); err != nil { - log.Error(err, "failed to create resource", "feature", f.Name, "name", m.name, "path", targetPath) + if err := applier(m.processedContent); err != nil { + f.Log.Error(err, "failed to create resource", "feature", f.Name, "name", m.name, "path", targetPath) return err } @@ -186,55 +211,31 @@ func (f *Feature) apply(m manifest) error { return nil } -func (f *Feature) OwnerReference() metav1.OwnerReference { +func (f *Feature) AsOwnerReference() metav1.OwnerReference { return f.Spec.Tracker.ToOwnerReference() } -// createResourceTracker instantiates FeatureTracker for a given Feature. All resources created when applying -// it will have this object attached as an OwnerReference. -// It's a cluster-scoped resource. Once created, there's a cleanup hook added which will be invoked on deletion, resulting -// in removal of all owned resources which belong to this Feature. -func (f *Feature) createResourceTracker() error { - tracker := &featurev1.FeatureTracker{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "features.opendatahub.io/v1", - Kind: "FeatureTracker", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: f.Spec.AppNamespace + "-" + common.TrimToRFC1123Name(f.Name), - }, - } - - foundTracker, err := f.DynamicClient.Resource(gvr.ResourceTracker).Get(context.TODO(), tracker.Name, metav1.GetOptions{}) - if k8serrors.IsNotFound(err) { - unstructuredTracker, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tracker) - if err != nil { - return err - } - - u := unstructured.Unstructured{Object: unstructuredTracker} - - foundTracker, err = f.DynamicClient.Resource(gvr.ResourceTracker).Create(context.TODO(), &u, metav1.CreateOptions{}) - if err != nil { - return err - } - } else if err != nil { - return err - } +// updateFeatureTrackerStatus updates conditions of a FeatureTracker. +// It's deliberately logging errors instead of handing them as it is used in deferred error handling of Feature public API, +// which is more predictable. +func (f *Feature) updateFeatureTrackerStatus(condType conditionsv1.ConditionType, status corev1.ConditionStatus, reason featurev1.FeaturePhase, message string) { + tracker := f.Spec.Tracker + + // Update the status + if tracker.Status.Conditions == nil { + tracker.Status.Conditions = &[]conditionsv1.Condition{} + } + conditionsv1.SetStatusCondition(tracker.Status.Conditions, conditionsv1.Condition{ + Type: condType, + Status: status, + Reason: string(reason), + Message: message, + }) - f.Spec.Tracker = &featurev1.FeatureTracker{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(foundTracker.Object, f.Spec.Tracker); err != nil { - return err + err := f.Client.Status().Update(context.Background(), tracker) + if err != nil { + f.Log.Error(err, "Error updating FeatureTracker status") } - // Register its own cleanup - f.addCleanup(func(feature *Feature) error { - if err := f.DynamicClient.Resource(gvr.ResourceTracker).Delete(context.TODO(), f.Spec.Tracker.Name, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { - return err - } - - return nil - }) - - return nil + f.Spec.Tracker.Status = tracker.Status } diff --git a/pkg/feature/feature_suite_test.go b/pkg/feature/feature_suite_test.go deleted file mode 100644 index 014ccee2b35..00000000000 --- a/pkg/feature/feature_suite_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package feature_test - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestFeatures(t *testing.T) { - RegisterFailHandler(Fail) - // for integration tests see tests/integration directory - RunSpecs(t, "Features unit tests") -} diff --git a/pkg/feature/feature_tracker_handler.go b/pkg/feature/feature_tracker_handler.go new file mode 100644 index 00000000000..e487847b0ae --- /dev/null +++ b/pkg/feature/feature_tracker_handler.go @@ -0,0 +1,94 @@ +package feature + +import ( + "context" + "fmt" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1" +) + +// createFeatureTracker instantiates FeatureTracker for a given Feature. It's a cluster-scoped resource used +// to track creation and removal of all owned resources which belong to this Feature. +// All resources which particular feature is composed of will have this object attached as an OwnerReference. +func (f *Feature) createFeatureTracker() error { + tracker, err := f.getFeatureTracker() + if k8serrors.IsNotFound(err) { + if err := f.Client.Create(context.TODO(), tracker); err != nil { + return err + } + } else if err != nil { + return err + } + + if err := f.ensureGVKSet(tracker); err != nil { + return err + } + + f.Spec.Tracker = tracker + + return nil +} + +func removeFeatureTracker(f *Feature) error { + if f.Spec.Tracker != nil { + return deleteTracker(f) + } + + if err := setFeatureTrackerIfAbsent(f); err != nil { + if k8serrors.IsNotFound(err) { + // There is nothing to delete + return nil + } + return err + } + + return deleteTracker(f) +} + +func (f *Feature) getFeatureTracker() (*featurev1.FeatureTracker, error) { + tracker := featurev1.NewFeatureTracker(f.Name, f.Spec.AppNamespace) + + tracker.Spec = featurev1.FeatureTrackerSpec{ + Source: *f.Spec.Source, + AppNamespace: f.Spec.AppNamespace, + } + + err := f.Client.Get(context.Background(), client.ObjectKeyFromObject(tracker), tracker) + + return tracker, err +} + +func setFeatureTrackerIfAbsent(f *Feature) error { + tracker, err := f.getFeatureTracker() + + f.Spec.Tracker = tracker + + return err +} + +func (f *Feature) ensureGVKSet(obj runtime.Object) error { + // See https://github.com/kubernetes/client-go/issues/308 + gvks, unversioned, err := f.Client.Scheme().ObjectKinds(obj) + if err != nil { + return fmt.Errorf("failed to get group, version, & kinds for object: %w", err) + } + if unversioned { + return fmt.Errorf("object is unversioned") + } + // Update the target object back with one of the discovered GVKs. + obj.GetObjectKind().SetGroupVersionKind(gvks[0]) + + return nil +} + +func deleteTracker(f *Feature) error { + err := f.Client.Delete(context.Background(), f.Spec.Tracker) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + return nil +} diff --git a/pkg/feature/handler.go b/pkg/feature/handler.go new file mode 100644 index 00000000000..0dc539a5bb1 --- /dev/null +++ b/pkg/feature/handler.go @@ -0,0 +1,69 @@ +package feature + +import ( + "fmt" + + "github.com/hashicorp/go-multierror" + + v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/components" +) + +// FeaturesHandler coordinates feature creations and removal from within controllers. +type FeaturesHandler struct { + *v1.DSCInitializationSpec + source featurev1.Source + features []*Feature + featuresProvider FeaturesProvider +} + +// FeaturesProvider is a function which allow to define list of features +// and couple them with the given initializer. +type FeaturesProvider func(handler *FeaturesHandler) error + +func ClusterFeaturesHandler(dsci *v1.DSCInitialization, def FeaturesProvider) *FeaturesHandler { + return &FeaturesHandler{ + DSCInitializationSpec: &dsci.Spec, + source: featurev1.Source{Type: featurev1.DSCIType, Name: dsci.Name}, + featuresProvider: def, + } +} + +func ComponentFeaturesHandler(component components.ComponentInterface, spec *v1.DSCInitializationSpec, def FeaturesProvider) *FeaturesHandler { + return &FeaturesHandler{ + DSCInitializationSpec: spec, + source: featurev1.Source{Type: featurev1.ComponentType, Name: component.GetComponentName()}, + featuresProvider: def, + } +} + +func (f *FeaturesHandler) Apply() error { + if err := f.featuresProvider(f); err != nil { + return fmt.Errorf("apply phase failed when wiring Feature instances: %w", err) + } + + var applyErrors *multierror.Error + for _, f := range f.features { + applyErrors = multierror.Append(applyErrors, f.Apply()) + } + + return applyErrors.ErrorOrNil() +} + +// Delete executes registered clean-up tasks in the opposite order they were initiated (following a stack structure). +// For instance, this allows for the undoing patches before its deletion. +// This approach assumes that Features are either instantiated in the correct sequence +// or are self-contained. +func (f *FeaturesHandler) Delete() error { + if err := f.featuresProvider(f); err != nil { + return fmt.Errorf("delete phase failed when wiring Feature instances: %w", err) + } + + var cleanupErrors *multierror.Error + for i := len(f.features) - 1; i >= 0; i-- { + cleanupErrors = multierror.Append(cleanupErrors, f.features[i].Cleanup()) + } + + return cleanupErrors.ErrorOrNil() +} diff --git a/pkg/feature/initializer.go b/pkg/feature/initializer.go deleted file mode 100644 index 3dfc86a4455..00000000000 --- a/pkg/feature/initializer.go +++ /dev/null @@ -1,55 +0,0 @@ -package feature - -import ( - "github.com/hashicorp/go-multierror" - - v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" -) - -type FeaturesInitializer struct { - *v1.DSCInitializationSpec - definedFeatures DefinedFeatures - Features []*Feature -} - -type DefinedFeatures func(s *FeaturesInitializer) error - -func NewFeaturesInitializer(spec *v1.DSCInitializationSpec, def DefinedFeatures) *FeaturesInitializer { - return &FeaturesInitializer{ - DSCInitializationSpec: spec, - definedFeatures: def, - } -} - -// Prepare performs validation of the spec and ensures all resources, -// such as Features and their templates, are processed and initialized -// before proceeding with the actual cluster set-up. -func (s *FeaturesInitializer) Prepare() error { - log.Info("Initializing features") - - return s.definedFeatures(s) -} - -func (s *FeaturesInitializer) Apply() error { - var applyErrors *multierror.Error - - for _, f := range s.Features { - applyErrors = multierror.Append(applyErrors, f.Apply()) - } - - return applyErrors.ErrorOrNil() -} - -// Delete executes registered clean-up tasks in the opposite order they were initiated (following a stack structure). -// For instance, this allows for the undoing patches before its deletion. -// This approach assumes that Features are either instantiated in the correct sequence -// or are self-contained. -func (s *FeaturesInitializer) Delete() error { - var cleanupErrors *multierror.Error - for i := len(s.Features) - 1; i >= 0; i-- { - log.Info("cleanup", "name", s.Features[i].Name) - cleanupErrors = multierror.Append(cleanupErrors, s.Features[i].Cleanup()) - } - - return cleanupErrors.ErrorOrNil() -} diff --git a/pkg/feature/manifest.go b/pkg/feature/manifest.go index e741c65b307..3fcbcd22de3 100644 --- a/pkg/feature/manifest.go +++ b/pkg/feature/manifest.go @@ -1,76 +1,111 @@ package feature import ( + "bytes" + "embed" "fmt" "html/template" - "os" + "io" + "io/fs" + "path" "path/filepath" "strings" - - "github.com/pkg/errors" ) -const BaseOutputDir = "/tmp/odh-operator" +//go:embed templates +var embeddedFiles embed.FS + +var ( + BaseDir = "templates" + ServiceMeshDir = path.Join(BaseDir, "servicemesh") + ServerlessDir = path.Join(BaseDir, "serverless") +) type manifest struct { name, - path string + path, + processedContent string template, - patch, - processed bool + patch bool + fsys fs.FS } -func loadManifestsFrom(path string) ([]manifest, error) { +func loadManifestsFrom(fsys fs.FS, path string) ([]manifest, error) { var manifests []manifest - if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + + err := fs.WalkDir(fsys, path, func(path string, dirEntry fs.DirEntry, walkErr error) error { + _, err := dirEntry.Info() if err != nil { return err } - if info.IsDir() { + + if dirEntry.IsDir() { return nil } - basePath := filepath.Base(path) - manifests = append(manifests, manifest{ - name: basePath, - path: path, - patch: strings.Contains(basePath, ".patch"), - template: filepath.Ext(path) == ".tmpl", - }) + m := createManifestFrom(fsys, path) + manifests = append(manifests, m) return nil - }); err != nil { - return nil, errors.WithStack(err) + }) + + if err != nil { + return nil, err } return manifests, nil } +func createManifestFrom(fsys fs.FS, path string) manifest { + basePath := filepath.Base(path) + m := manifest{ + name: basePath, + path: path, + patch: strings.Contains(basePath, ".patch"), + template: filepath.Ext(path) == ".tmpl", + fsys: fsys, + } + + return m +} + func (m *manifest) targetPath() string { return fmt.Sprintf("%s%s", m.path[:len(m.path)-len(filepath.Ext(m.path))], ".yaml") } -func (m *manifest) processTemplate(data interface{}) error { - if !m.template { - return nil +func (m *manifest) process(data interface{}) error { + manifestFile, err := m.open() + if err != nil { + return err } - path := m.targetPath() + defer manifestFile.Close() - f, err := os.Create(path) + content, err := io.ReadAll(manifestFile) if err != nil { - log.Error(err, "Failed to create file") - - return err + return fmt.Errorf("failed to create file: %w", err) } - tmpl := template.New(m.name).Funcs(template.FuncMap{"ReplaceChar": ReplaceChar}) + if !m.template { + // If, by convention, the file is not suffixed with `.tmpl` we do not need to trigger template processing. + // It's safe to return at this point. + m.processedContent = string(content) + return nil + } - tmpl, err = tmpl.ParseFiles(m.path) + tmpl, err := template.New(m.name).Funcs(template.FuncMap{"ReplaceChar": ReplaceChar}).Parse(string(content)) if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + + var buffer bytes.Buffer + if err := tmpl.Execute(&buffer, data); err != nil { return err } - err = tmpl.Execute(f, data) - m.processed = err == nil + m.processedContent = buffer.String() + + return nil +} - return err +func (m *manifest) open() (fs.File, error) { + return m.fsys.Open(m.path) } diff --git a/pkg/feature/raw_resources.go b/pkg/feature/raw_resources.go index bf3abea8e8b..73468c78082 100644 --- a/pkg/feature/raw_resources.go +++ b/pkg/feature/raw_resources.go @@ -16,7 +16,6 @@ package feature import ( "context" "fmt" - "os" "regexp" "strings" @@ -33,13 +32,9 @@ const ( YamlSeparator = "(?m)^---[ \t]*$" ) -func (f *Feature) createResourceFromFile(filename string) error { - data, err := os.ReadFile(filename) - if err != nil { - return errors.WithStack(err) - } +func (f *Feature) createResources(resources string) error { splitter := regexp.MustCompile(YamlSeparator) - objectStrings := splitter.Split(string(data), -1) + objectStrings := splitter.Split(resources, -1) for _, str := range objectStrings { if strings.TrimSpace(str) == "" { continue @@ -55,14 +50,14 @@ func (f *Feature) createResourceFromFile(filename string) error { namespace := u.GetNamespace() u.SetOwnerReferences([]metav1.OwnerReference{ - f.OwnerReference(), + f.AsOwnerReference(), }) - log.Info("Creating resource", "name", name) + f.Log.Info("Creating resource", "name", name) err := f.Client.Get(context.TODO(), k8stypes.NamespacedName{Name: name, Namespace: namespace}, u.DeepCopy()) if err == nil { - log.Info("Object already exists...") + f.Log.Info("Object already exists...") continue } @@ -79,20 +74,16 @@ func (f *Feature) createResourceFromFile(filename string) error { return nil } -func (f *Feature) patchResourceFromFile(filename string) error { - data, err := os.ReadFile(filename) - if err != nil { - return errors.WithStack(err) - } +func (f *Feature) patchResources(resources string) error { splitter := regexp.MustCompile(YamlSeparator) - objectStrings := splitter.Split(string(data), -1) + objectStrings := splitter.Split(resources, -1) for _, str := range objectStrings { if strings.TrimSpace(str) == "" { continue } u := &unstructured.Unstructured{} if err := yaml.Unmarshal([]byte(str), u); err != nil { - log.Error(err, "error unmarshalling yaml") + f.Log.Error(err, "error unmarshalling yaml") return errors.WithStack(err) } @@ -105,10 +96,10 @@ func (f *Feature) patchResourceFromFile(filename string) error { Resource: strings.ToLower(u.GroupVersionKind().Kind) + "s", } - // Convert the patch from YAML to JSON - patchAsJSON, err := yaml.YAMLToJSON(data) + // Convert the individual resource patch from YAML to JSON + patchAsJSON, err := yaml.YAMLToJSON([]byte(str)) if err != nil { - log.Error(err, "error converting yaml to json") + f.Log.Error(err, "error converting yaml to json") return errors.WithStack(err) } @@ -117,7 +108,7 @@ func (f *Feature) patchResourceFromFile(filename string) error { Namespace(u.GetNamespace()). Patch(context.TODO(), u.GetName(), k8stypes.MergePatchType, patchAsJSON, metav1.PatchOptions{}) if err != nil { - log.Error(err, "error patching resource", + f.Log.Error(err, "error patching resource", "gvr", fmt.Sprintf("%+v\n", gvr), "patch", fmt.Sprintf("%+v\n", u), "json", fmt.Sprintf("%+v\n", patchAsJSON)) diff --git a/pkg/feature/serverless/conditions.go b/pkg/feature/serverless/conditions.go index 12714fc0bf5..739e9ebd2e1 100644 --- a/pkg/feature/serverless/conditions.go +++ b/pkg/feature/serverless/conditions.go @@ -11,6 +11,10 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" ) +const ( + KnativeServingNamespace = "knative-serving" +) + var log = ctrlLog.Log.WithName("features") func EnsureServerlessAbsent(f *feature.Feature) error { @@ -28,7 +32,7 @@ func EnsureServerlessAbsent(f *feature.Feature) error { } servingOwners := list.Items[0].GetOwnerReferences() - featureOwner := f.OwnerReference() + featureOwner := f.AsOwnerReference() for _, owner := range servingOwners { if owner.APIVersion == featureOwner.APIVersion && owner.Kind == featureOwner.Kind && @@ -50,3 +54,5 @@ func EnsureServerlessOperatorInstalled(f *feature.Feature) error { return nil } + +var EnsureServerlessServingDeployed = feature.WaitForResourceToBeCreated(KnativeServingNamespace, gvr.KnativeServing) diff --git a/pkg/feature/servicemesh/conditions.go b/pkg/feature/servicemesh/conditions.go index da8e17e51e9..e352c9545a7 100644 --- a/pkg/feature/servicemesh/conditions.go +++ b/pkg/feature/servicemesh/conditions.go @@ -2,6 +2,7 @@ package servicemesh import ( "context" + "fmt" "time" "github.com/hashicorp/go-multierror" @@ -10,14 +11,11 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" - ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" ) -var log = ctrlLog.Log.WithName("features") - const ( interval = 2 * time.Second duration = 5 * time.Minute @@ -25,7 +23,7 @@ const ( func EnsureServiceMeshOperatorInstalled(f *feature.Feature) error { if err := feature.EnsureCRDIsInstalled("servicemeshcontrolplanes.maistra.io")(f); err != nil { - log.Info("Failed to find the pre-requisite Service Mesh Control Plane CRD, please ensure Service Mesh Operator is installed.", "feature", f.Name) + f.Log.Info("Failed to find the pre-requisite Service Mesh Control Plane CRD, please ensure Service Mesh Operator is installed.") return err } @@ -42,7 +40,7 @@ func EnsureServiceMeshInstalled(f *feature.Feature) error { smcpNs := f.Spec.ControlPlane.Namespace if err := WaitForControlPlaneToBeReady(f); err != nil { - log.Error(err, "failed waiting for control plane being ready", "feature", f.Name, "control-plane", smcp, "namespace", smcpNs) + f.Log.Error(err, "failed waiting for control plane being ready", "control-plane", smcp, "namespace", smcpNs) return multierror.Append(err, errors.New("service mesh control plane is not ready")).ErrorOrNil() } @@ -50,17 +48,17 @@ func EnsureServiceMeshInstalled(f *feature.Feature) error { return nil } -func WaitForControlPlaneToBeReady(feature *feature.Feature) error { - smcp := feature.Spec.ControlPlane.Name - smcpNs := feature.Spec.ControlPlane.Namespace +func WaitForControlPlaneToBeReady(f *feature.Feature) error { + smcp := f.Spec.ControlPlane.Name + smcpNs := f.Spec.ControlPlane.Namespace - log.Info("waiting for control plane components to be ready", "feature", feature.Name, "control-plane", smcp, "namespace", smcpNs, "duration (s)", duration.Seconds()) + f.Log.Info("waiting for control plane components to be ready", "control-plane", smcp, "namespace", smcpNs, "duration (s)", duration.Seconds()) return wait.PollUntilContextTimeout(context.TODO(), interval, duration, false, func(ctx context.Context) (bool, error) { - ready, err := CheckControlPlaneComponentReadiness(feature.DynamicClient, smcp, smcpNs) + ready, err := CheckControlPlaneComponentReadiness(f.DynamicClient, smcp, smcpNs) if ready { - log.Info("done waiting for control plane components to be ready", "feature", feature.Name, "control-plane", smcp, "namespace", smcpNs) + f.Log.Info("done waiting for control plane components to be ready", "control-plane", smcp, "namespace", smcpNs) } return ready, err @@ -70,19 +68,17 @@ func WaitForControlPlaneToBeReady(feature *feature.Feature) error { func CheckControlPlaneComponentReadiness(dynamicClient dynamic.Interface, smcp, smcpNs string) (bool, error) { unstructObj, err := dynamicClient.Resource(gvr.SMCP).Namespace(smcpNs).Get(context.TODO(), smcp, metav1.GetOptions{}) if err != nil { - log.Info("failed to find Service Mesh Control Plane", "control-plane", smcp, "namespace", smcpNs) - return false, err + return false, fmt.Errorf("failed to find Service Mesh Control Plane: %w", err) } components, found, err := unstructured.NestedMap(unstructObj.Object, "status", "readiness", "components") if err != nil || !found { - log.Info("status conditions not found or error in parsing of Service Mesh Control Plane") - return false, err + return false, fmt.Errorf("status conditions not found or error in parsing of Service Mesh Control Plane: %w", err) } - readyComponents := len(components["ready"].([]interface{})) - pendingComponents := len(components["pending"].([]interface{})) - unreadyComponents := len(components["unready"].([]interface{})) + readyComponents := len(components["ready"].([]interface{})) //nolint:forcetypeassert + pendingComponents := len(components["pending"].([]interface{})) //nolint:forcetypeassert + unreadyComponents := len(components["unready"].([]interface{})) //nolint:forcetypeassert return pendingComponents == 0 && unreadyComponents == 0 && readyComponents > 0, nil } diff --git a/pkg/feature/template_loader.go b/pkg/feature/template_loader.go deleted file mode 100644 index 9d2fd53cf1e..00000000000 --- a/pkg/feature/template_loader.go +++ /dev/null @@ -1,39 +0,0 @@ -package feature - -import ( - "embed" - "io/fs" - "os" - "path/filepath" -) - -//go:embed templates -var embeddedFiles embed.FS - -// CopyEmbeddedFiles ensures that files embedded using go:embed are populated -// to dest directory. In order to process the templates, we need to create a tmp directory -// to store the files. This is because embedded files are read only. -func CopyEmbeddedFiles(src, dest string) error { - return fs.WalkDir(embeddedFiles, src, func(path string, dir fs.DirEntry, err error) error { - if err != nil { - return err - } - - destPath := filepath.Join(dest, path) - if dir.IsDir() { - if err := os.MkdirAll(destPath, 0755); err != nil { - return err - } - } else { - data, err := fs.ReadFile(embeddedFiles, path) - if err != nil { - return err - } - if err := os.WriteFile(destPath, data, 0644); err != nil { - return err - } - } - - return nil - }) -} diff --git a/pkg/feature/templates/serverless/serving-install/knative-serving.tmpl b/pkg/feature/templates/serverless/serving-install/knative-serving.tmpl index ae4c947b398..58f8bc8a488 100644 --- a/pkg/feature/templates/serverless/serving-install/knative-serving.tmpl +++ b/pkg/feature/templates/serverless/serving-install/knative-serving.tmpl @@ -6,7 +6,7 @@ metadata: annotations: serverless.openshift.io/default-enable-http2: "true" spec: - deployments: + workloads: - annotations: sidecar.istio.io/inject: "true" sidecar.istio.io/rewriteAppHTTPProbers: "true" diff --git a/pkg/feature/templates/serverless/serving-istio-gateways/istio-local-gateway.tmpl b/pkg/feature/templates/serverless/serving-istio-gateways/istio-local-gateway.yaml similarity index 100% rename from pkg/feature/templates/serverless/serving-istio-gateways/istio-local-gateway.tmpl rename to pkg/feature/templates/serverless/serving-istio-gateways/istio-local-gateway.yaml diff --git a/pkg/feature/templates/serverless/serving-net-istio-secret-filtering.patch.tmpl b/pkg/feature/templates/serverless/serving-net-istio-secret-filtering.patch.tmpl new file mode 100644 index 00000000000..5153cbc0fa9 --- /dev/null +++ b/pkg/feature/templates/serverless/serving-net-istio-secret-filtering.patch.tmpl @@ -0,0 +1,21 @@ +apiVersion: operator.knative.dev/v1beta1 +kind: KnativeServing +metadata: + name: {{ .Serving.Name }} + namespace: knative-serving +spec: + workloads: + - name: activator + annotations: + sidecar.istio.io/inject: "true" + sidecar.istio.io/rewriteAppHTTPProbers: "true" + - name: autoscaler + annotations: + sidecar.istio.io/inject: "true" + sidecar.istio.io/rewriteAppHTTPProbers: "true" + - name: net-istio-controller + env: + - container: controller + envVars: + - name: ENABLE_SECRET_INFORMER_FILTERING_BY_CERT_UID + value: 'true' diff --git a/pkg/feature/templates/servicemesh/base/smcp.tmpl b/pkg/feature/templates/servicemesh/base/create-smcp.tmpl similarity index 96% rename from pkg/feature/templates/servicemesh/base/smcp.tmpl rename to pkg/feature/templates/servicemesh/base/create-smcp.tmpl index eb9aead9f3c..8a7ed5a22d7 100644 --- a/pkg/feature/templates/servicemesh/base/smcp.tmpl +++ b/pkg/feature/templates/servicemesh/base/create-smcp.tmpl @@ -19,6 +19,8 @@ spec: security: dataPlane: mtls: true # otherwise inference-graph will not work. We use PeerAuthentication resources to force mTLS + identity: + type: ThirdParty techPreview: meshConfig: defaultConfig: diff --git a/pkg/feature/types.go b/pkg/feature/types.go index 7b5da4cc7b8..4857bb1d243 100644 --- a/pkg/feature/types.go +++ b/pkg/feature/types.go @@ -16,6 +16,7 @@ type Spec struct { KnativeCertificateSecret string KnativeIngressDomain string Tracker *featurev1.FeatureTracker + Source *featurev1.Source } type OAuth struct { diff --git a/pkg/gvr/gvr.go b/pkg/gvr/gvr.go index c970b82d3bd..c8066ea4a12 100644 --- a/pkg/gvr/gvr.go +++ b/pkg/gvr/gvr.go @@ -15,7 +15,7 @@ var ( Resource: "ingresses", } - ResourceTracker = schema.GroupVersionResource{ + FeatureTracker = schema.GroupVersionResource{ Group: "features.opendatahub.io", Version: "v1", Resource: "featuretrackers", @@ -26,4 +26,10 @@ var ( Version: "v2", Resource: "servicemeshcontrolplanes", } + + NetworkPolicies = schema.GroupVersionResource{ + Group: "networking.k8s.io", + Version: "v1", + Resource: "networkpolicies", + } ) diff --git a/pkg/monitoring/monitoring.go b/pkg/monitoring/monitoring.go new file mode 100644 index 00000000000..67a71d85cac --- /dev/null +++ b/pkg/monitoring/monitoring.go @@ -0,0 +1,45 @@ +package monitoring + +import ( + "context" + "fmt" + "time" + + errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// WaitForDeploymentAvailable to check if component deployment from 'namepsace' is ready within 'timeout' before apply prometheus rules for the component. +func WaitForDeploymentAvailable(_ context.Context, restConfig *rest.Config, componentName string, namespace string, interval int, timeout int) error { + resourceInterval := time.Duration(interval) * time.Second + resourceTimeout := time.Duration(timeout) * time.Minute + return wait.PollUntilContextTimeout(context.TODO(), resourceInterval, resourceTimeout, true, func(ctx context.Context) (bool, error) { + clientset, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return false, fmt.Errorf("error getting client %w", err) + } + componentDeploymentList, err := clientset.AppsV1().Deployments(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app.opendatahub.io/" + componentName, + }) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + } + isReady := false + fmt.Printf("waiting for %d deployment to be ready for %s\n", len(componentDeploymentList.Items), componentName) + if len(componentDeploymentList.Items) != 0 { + for _, deployment := range componentDeploymentList.Items { + if deployment.Status.ReadyReplicas == deployment.Status.Replicas { + isReady = true + } else { + isReady = false + } + } + } + return isReady, nil + }) +} diff --git a/pkg/plugins/addLabelsplugin.go b/pkg/plugins/addLabelsplugin.go index e4cf31883eb..52d75964c5a 100644 --- a/pkg/plugins/addLabelsplugin.go +++ b/pkg/plugins/addLabelsplugin.go @@ -1,7 +1,7 @@ package plugins import ( - "sigs.k8s.io/kustomize/api/builtins" //nolint + "sigs.k8s.io/kustomize/api/builtins" //nolint:staticcheck //Remove after package update "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/resid" @@ -11,6 +11,7 @@ func ApplyAddLabelsPlugin(componentName string, resMap resmap.ResMap) error { nsplug := builtins.LabelTransformerPlugin{ Labels: map[string]string{ "app.opendatahub.io/" + componentName: "true", + "app.kubernetes.io/part-of": componentName, }, FieldSpecs: []types.FieldSpec{ { @@ -18,6 +19,11 @@ func ApplyAddLabelsPlugin(componentName string, resMap resmap.ResMap) error { Path: "spec/template/metadata/labels", CreateIfNotPresent: true, }, + { + Gvk: resid.Gvk{Kind: "Deployment"}, + Path: "spec/selector/matchLabels", + CreateIfNotPresent: true, + }, { Gvk: resid.Gvk{}, Path: "metadata/labels", diff --git a/pkg/plugins/namespacePlugin.go b/pkg/plugins/namespacePlugin.go index c9e8846aa14..3b04601096e 100644 --- a/pkg/plugins/namespacePlugin.go +++ b/pkg/plugins/namespacePlugin.go @@ -1,7 +1,7 @@ package plugins import ( - "sigs.k8s.io/kustomize/api/builtins" //nolint + "sigs.k8s.io/kustomize/api/builtins" //nolint:staticcheck // Remove after package update "sigs.k8s.io/kustomize/api/filters/namespace" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/types" diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index e2602aa5806..99c1406aa88 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -2,7 +2,6 @@ package upgrade import ( "context" - "encoding/json" "fmt" "os" "strings" @@ -12,11 +11,12 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" ofapi "github.com/operator-framework/api/pkg/operators/v1alpha1" olmclientset "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1alpha1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,10 +28,13 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/components/dashboard" "github.com/opendatahub-io/opendatahub-operator/v2/components/datasciencepipelines" "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" + "github.com/opendatahub-io/opendatahub-operator/v2/components/kueue" "github.com/opendatahub-io/opendatahub-operator/v2/components/modelmeshserving" + "github.com/opendatahub-io/opendatahub-operator/v2/components/modelregistry" "github.com/opendatahub-io/opendatahub-operator/v2/components/ray" "github.com/opendatahub-io/opendatahub-operator/v2/components/trustyai" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" + infrav1 "github.com/opendatahub-io/opendatahub-operator/v2/infrastructure/v1" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" ) @@ -65,7 +68,7 @@ func OperatorUninstall(cli client.Client, cfg *rest.Config) error { client.MatchingLabels{cluster.ODHGeneratedNamespaceLabel: "true"}, } if err := cli.List(context.TODO(), generatedNamespaces, nsOptions...); err != nil { - return fmt.Errorf("error getting generated namespaces : %v", err) + return fmt.Errorf("error getting generated namespaces : %w", err) } // Return if any one of the namespaces is Terminating due to resources that are in process of deletion. (e.g CRDs) @@ -76,9 +79,10 @@ func OperatorUninstall(cli client.Client, cfg *rest.Config) error { } for _, namespace := range generatedNamespaces.Items { + namespace := namespace if namespace.Status.Phase == corev1.NamespaceActive { if err := cli.Delete(context.TODO(), &namespace, []client.DeleteOption{}...); err != nil { - return fmt.Errorf("error deleting namespace %v: %v", namespace.Name, err) + return fmt.Errorf("error deleting namespace %v: %w", namespace.Name, err) } fmt.Printf("Namespace %s deleted as a part of uninstall.\n", namespace.Name) } @@ -101,6 +105,7 @@ func removeDSCInitialization(cli client.Client) error { var multiErr *multierror.Error for _, dsciInstance := range instanceList.Items { + dsciInstance := dsciInstance if err := cli.Delete(context.TODO(), &dsciInstance); !apierrs.IsNotFound(err) { multiErr = multierror.Append(multiErr, err) } @@ -134,7 +139,7 @@ func HasDeleteConfigMap(c client.Client) bool { // createDefaultDSC creates a default instance of DSC. // Note: When the platform is not Managed, and a DSC instance already exists, the function doesn't re-create/update the resource. -func CreateDefaultDSC(cli client.Client, platform deploy.Platform) error { +func CreateDefaultDSC(cli client.Client, _ deploy.Platform) error { // Set the default DSC name depending on the platform releaseDataScienceCluster := &dsc.DataScienceCluster{ TypeMeta: metav1.TypeMeta{ @@ -159,7 +164,7 @@ func CreateDefaultDSC(cli client.Client, platform deploy.Platform) error { Component: components.Component{ManagementState: operatorv1.Managed}, }, Kserve: kserve.Kserve{ - Component: components.Component{ManagementState: operatorv1.Removed}, + Component: components.Component{ManagementState: operatorv1.Managed}, }, CodeFlare: codeflare.CodeFlare{ Component: components.Component{ManagementState: operatorv1.Removed}, @@ -167,7 +172,13 @@ func CreateDefaultDSC(cli client.Client, platform deploy.Platform) error { Ray: ray.Ray{ Component: components.Component{ManagementState: operatorv1.Removed}, }, + Kueue: kueue.Kueue{ + Component: components.Component{ManagementState: operatorv1.Removed}, + }, TrustyAI: trustyai.TrustyAI{ + Component: components.Component{ManagementState: operatorv1.Managed}, + }, + ModelRegistry: modelregistry.ModelRegistry{ Component: components.Component{ManagementState: operatorv1.Removed}, }, }, @@ -182,7 +193,7 @@ func CreateDefaultDSC(cli client.Client, platform deploy.Platform) error { fmt.Printf("DataScienceCluster resource already exists. It will not be updated with default DSC.\n") return nil default: - return fmt.Errorf("failed to create DataScienceCluster custom resource: %v", err) + return fmt.Errorf("failed to create DataScienceCluster custom resource: %w", err) } return nil @@ -190,14 +201,22 @@ func CreateDefaultDSC(cli client.Client, platform deploy.Platform) error { // createDefaultDSCI creates a default instance of DSCI // If there exists an instance already, it patches the DSCISpec with default values -// Note: DSCI CR modifcations are not supported, as it is the initial prereq setting for the components -func CreateDefaultDSCI(cli client.Client, platform deploy.Platform, appNamespace, monNamespace string) error { +// Note: DSCI CR modifcations are not supported, as it is the initial prereq setting for the components. +func CreateDefaultDSCI(cli client.Client, _ deploy.Platform, appNamespace, monNamespace string) error { defaultDsciSpec := &dsci.DSCInitializationSpec{ ApplicationsNamespace: appNamespace, Monitoring: dsci.Monitoring{ ManagementState: operatorv1.Managed, Namespace: monNamespace, }, + ServiceMesh: infrav1.ServiceMeshSpec{ + ManagementState: "Managed", + ControlPlane: infrav1.ControlPlaneSpec{ + Name: "data-science-smcp", + Namespace: "istio-system", + MetricsCollection: "Istio", + }, + }, } defaultDsci := &dsci.DSCInitialization{ @@ -211,14 +230,6 @@ func CreateDefaultDSCI(cli client.Client, platform deploy.Platform, appNamespace Spec: *defaultDsciSpec, } - patchedDSCI := &dsci.DSCInitialization{ - TypeMeta: metav1.TypeMeta{ - Kind: "DSCInitialization", - APIVersion: "dscinitialization.opendatahub.io/v1", - }, - Spec: *defaultDsciSpec, - } - instances := &dsci.DSCInitializationList{} if err := cli.List(context.TODO(), instances); err != nil { return err @@ -229,21 +240,11 @@ func CreateDefaultDSCI(cli client.Client, platform deploy.Platform, appNamespace fmt.Printf("only one instance of DSCInitialization object is allowed. Please delete other instances.\n") return nil case len(instances.Items) == 1: - if platform == deploy.ManagedRhods || platform == deploy.SelfManagedRhods { - data, err := json.Marshal(patchedDSCI) - if err != nil { - return err - } - existingDSCI := &instances.Items[0] - err = cli.Patch(context.TODO(), existingDSCI, client.RawPatch(types.ApplyPatchType, data), - client.ForceOwnership, client.FieldOwner("opendatahub-operator")) - if err != nil { - return err - } - } else { - return nil - } + // Do not patch/update if DSCI already exists. + fmt.Printf("DSCInitialization resource already exists. It will not be updated with default DSCI.") + return nil case len(instances.Items) == 0: + fmt.Printf("create default DSCI CR.") err := cli.Create(context.TODO(), defaultDsci) if err != nil { return err @@ -252,31 +253,64 @@ func CreateDefaultDSCI(cli client.Client, platform deploy.Platform, appNamespace return nil } -func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform) error { +func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform, appNS string, montNamespace string) error { // If platform is Managed, remove Kfdefs and create default dsc if platform == deploy.ManagedRhods { - err := CreateDefaultDSC(cli, platform) - if err != nil { + fmt.Println("starting deletion of Deployment in managed cluster") + if err := deleteResource(cli, appNS, "deployment"); err != nil { return err } - - err = RemoveKfDefInstances(cli, platform) - if err != nil { + // this is for the modelmesh monitoring part from v1 to v2 + if err := deleteResource(cli, montNamespace, "deployment"); err != nil { return err } - - return nil + if err := deleteResource(cli, montNamespace, "statefulset"); err != nil { + return err + } + fmt.Println("creating default DSC CR") + if err := CreateDefaultDSC(cli, platform); err != nil { + return err + } + return RemoveKfDefInstances(cli, platform) } if platform == deploy.SelfManagedRhods { - kfDefList, err := getKfDefInstances(cli) - if err != nil { - return fmt.Errorf("error getting kfdef instances: %v", err) + fmt.Println("starting deletion of Deployment in selfmanaged cluster") + // If KfDef CRD is not found, we see it as a cluster not pre-installed v1 operator // Check if kfdef are deployed + kfdefCrd := &apiextv1.CustomResourceDefinition{} + if err := cli.Get(context.TODO(), client.ObjectKey{Name: "kfdefs.kfdef.apps.kubeflow.org"}, kfdefCrd); err != nil { + if apierrs.IsNotFound(err) { + // If no Crd found, return, since its a new Installation + // return empty list + return nil + } + return fmt.Errorf("error retrieving kfdef CRD : %w", err) } + // If KfDef Instances found, and no DSC instances are found in Self-managed, that means this is an upgrade path from + // legacy version. Create a default DSC instance + kfDefList := &kfdefv1.KfDefList{} + err := cli.List(context.TODO(), kfDefList) + if err != nil { + if apierrs.IsNotFound(err) { + // If no KfDefs, do nothing and return + return nil + } + return fmt.Errorf("error getting kfdef instances: : %w", err) + } if len(kfDefList.Items) > 0 { - err := CreateDefaultDSC(cli, platform) - if err != nil { + if err = deleteResource(cli, appNS, "deployment"); err != nil { + return fmt.Errorf("error deleting deployment: %w", err) + } + // this is for the modelmesh monitoring part from v1 to v2 + if err := deleteResource(cli, montNamespace, "deployment"); err != nil { + return err + } + if err := deleteResource(cli, montNamespace, "statefulset"); err != nil { + return err + } + // create default DSC + if err = CreateDefaultDSC(cli, platform); err != nil { return err } } @@ -284,6 +318,14 @@ func UpdateFromLegacyVersion(cli client.Client, platform deploy.Platform) error return err } + // TODO: Revert the following condition in 2.8 ODH Release + if platform == deploy.OpenDataHub { + fmt.Println("starting deletion of deployment in ODH cluster") + if err := deleteResource(cli, appNS, "deployment"); err != nil { + return fmt.Errorf("error deleting deployment: %w", err) + } + } + return nil } @@ -298,29 +340,42 @@ func GetOperatorNamespace() (string, error) { return "", err } -func RemoveKfDefInstances(cli client.Client, platform deploy.Platform) error { +func RemoveKfDefInstances(cli client.Client, _ deploy.Platform) error { // Check if kfdef are deployed - expectedKfDefList, err := getKfDefInstances(cli) + kfdefCrd := &apiextv1.CustomResourceDefinition{} + + err := cli.Get(context.TODO(), client.ObjectKey{Name: "kfdefs.kfdef.apps.kubeflow.org"}, kfdefCrd) if err != nil { - return err + if apierrs.IsNotFound(err) { + // If no Crd found, return, since its a new Installation + return nil + } + return fmt.Errorf("error retrieving kfdef CRD : %w", err) + } + expectedKfDefList := &kfdefv1.KfDefList{} + err = cli.List(context.TODO(), expectedKfDefList) + if err != nil { + if apierrs.IsNotFound(err) { + // If no KfDefs, do nothing and return + return nil + } + return fmt.Errorf("error getting list of kfdefs: %w", err) } // Delete kfdefs - if len(expectedKfDefList.Items) > 0 { - for _, kfdef := range expectedKfDefList.Items { - // Remove finalizer - updatedKfDef := &kfdef - updatedKfDef.Finalizers = []string{} - err = cli.Update(context.TODO(), updatedKfDef) - if err != nil { - return fmt.Errorf("error removing finalizers from kfdef %v : %v", kfdef.Name, err) - } - err = cli.Delete(context.TODO(), updatedKfDef) - if err != nil { - return fmt.Errorf("error deleting kfdef %v : %v", kfdef.Name, err) - } + for _, kfdef := range expectedKfDefList.Items { + kfdef := kfdef + // Remove finalizer + updatedKfDef := &kfdef + updatedKfDef.Finalizers = []string{} + err = cli.Update(context.TODO(), updatedKfDef) + if err != nil { + return fmt.Errorf("error removing finalizers from kfdef %v : %w", kfdef.Name, err) + } + err = cli.Delete(context.TODO(), updatedKfDef) + if err != nil { + return fmt.Errorf("error deleting kfdef %v : %w", kfdef.Name, err) } } - return nil } @@ -344,7 +399,7 @@ func removeCsv(c client.Client, r *rest.Config) error { return nil } - return fmt.Errorf("error deleting clusterserviceversion: %v", err) + return fmt.Errorf("error deleting clusterserviceversion: %w", err) } fmt.Printf("Clusterserviceversion %s deleted as a part of uninstall.\n", operatorCsv.Name) } @@ -357,7 +412,7 @@ func removeCsv(c client.Client, r *rest.Config) error { func getClusterServiceVersion(cfg *rest.Config, watchNameSpace string) (*ofapi.ClusterServiceVersion, error) { operatorClient, err := olmclientset.NewForConfig(cfg) if err != nil { - return nil, fmt.Errorf("error getting operator client %v", err) + return nil, fmt.Errorf("error getting operator client %w", err) } csvs, err := operatorClient.ClusterServiceVersions(watchNameSpace).List(context.TODO(), metav1.ListOptions{}) if err != nil { @@ -378,26 +433,141 @@ func getClusterServiceVersion(cfg *rest.Config, watchNameSpace string) (*ofapi.C return nil, nil } -func getKfDefInstances(c client.Client) (*kfdefv1.KfDefList, error) { - // If KfDef CRD is not found, we see it as a cluster not pre-installed v1 operator - // Check if kfdef are deployed - kfdefCrd := &apiextv1.CustomResourceDefinition{} - if err := c.Get(context.TODO(), client.ObjectKey{Name: "kfdefs.kfdef.apps.kubeflow.org"}, kfdefCrd); err != nil { - if apierrs.IsNotFound(err) { - // If no Crd found, return, since its a new Installation - // return empty list - return &kfdefv1.KfDefList{}, nil - } else { - return nil, fmt.Errorf("error retrieving kfdef CRD : %v", err) +func deleteResource(cli client.Client, namespace string, resourceType string) error { + // In v2, Deployment selectors use a label "app.opendatahub.io/" which is + // not present in v1. Since label selectors are immutable, we need to delete the existing + // deployments and recreated them. + // because we can't proceed if a deployment is not deleted, we use exponential backoff + // to retry the deletion until it succeeds + var err error + switch resourceType { + case "deployment": + err = wait.ExponentialBackoffWithContext(context.TODO(), wait.Backoff{ + // 5, 10, ,20, 40 then timeout + Duration: 5 * time.Second, + Factor: 2.0, + Jitter: 0.1, + Steps: 4, + Cap: 1 * time.Minute, + }, func(ctx context.Context) (bool, error) { + done, err := deleteDeploymentsAndCheck(ctx, cli, namespace) + return done, err + }) + case "statefulset": + err = wait.ExponentialBackoffWithContext(context.TODO(), wait.Backoff{ + // 10, 20 then timeout + Duration: 10 * time.Second, + Factor: 2.0, + Jitter: 0.1, + Steps: 2, + Cap: 1 * time.Minute, + }, func(ctx context.Context) (bool, error) { + done, err := deleteStatefulsetsAndCheck(ctx, cli, namespace) + return done, err + }) + } + return err +} + +func deleteDeploymentsAndCheck(ctx context.Context, cli client.Client, namespace string) (bool, error) { + // Delete Deployment objects + var multiErr *multierror.Error + deployments := &appsv1.DeploymentList{} + listOpts := &client.ListOptions{ + Namespace: namespace, + } + + if err := cli.List(ctx, deployments, listOpts); err != nil { + return false, nil //nolint:nilerr + } + // filter deployment which has the new label to limit that we do not over kill other deployment + // this logic can be used even when upgrade from v2.4 to v2.5 without remove it + markedForDeletion := []appsv1.Deployment{} + for _, deployment := range deployments.Items { + deployment := deployment + v2 := false + selectorLabels := deployment.Spec.Selector.MatchLabels + for label := range selectorLabels { + if strings.Contains(label, "app.opendatahub.io/") { + // this deployment has the new label, this is a v2 to v2 upgrade + // there is no need to recreate it, as labels are matching + v2 = true + continue + } + } + if !v2 { + markedForDeletion = append(markedForDeletion, deployment) + multiErr = multierror.Append(multiErr, cli.Delete(ctx, &deployment)) + } + } + + for _, deployment := range markedForDeletion { + deployment := deployment + if e := cli.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: deployment.Name, + }, &deployment); e != nil { + if apierrs.IsNotFound(e) { + // resource has been successfully deleted + continue + } + // unexpected error, report it + multiErr = multierror.Append(multiErr, e) //nolint:staticcheck,wastedassign + } + // resource still exists, wait for it to be deleted + return false, nil + } + + return true, multiErr.ErrorOrNil() +} + +func deleteStatefulsetsAndCheck(ctx context.Context, cli client.Client, namespace string) (bool, error) { + // Delete statefulset objects + var multiErr *multierror.Error + statefulsets := &appsv1.StatefulSetList{} + listOpts := &client.ListOptions{ + Namespace: namespace, + } + + if err := cli.List(ctx, statefulsets, listOpts); err != nil { + return false, nil //nolint:nilerr + } + + // even only we have one item to delete to avoid nil point still use range + markedForDeletion := []appsv1.StatefulSet{} + for _, statefulset := range statefulsets.Items { + v2 := false + statefulset := statefulset + selectorLabels := statefulset.Spec.Selector.MatchLabels + for label := range selectorLabels { + if strings.Contains(label, "app.opendatahub.io/") { + v2 = true + continue + } + } + if !v2 { + markedForDeletion = append(markedForDeletion, statefulset) + multiErr = multierror.Append(multiErr, cli.Delete(ctx, &statefulset)) } } - // If KfDef Instances found, and no DSC instances are found in Self-managed, that means this is an upgrade path from - // legacy version. Create a default DSC instance - kfDefList := &kfdefv1.KfDefList{} - if err := c.List(context.TODO(), kfDefList); err != nil { - return &kfdefv1.KfDefList{}, fmt.Errorf("error getting list of kfdefs: %v", err) + for _, statefulset := range markedForDeletion { + statefulset := statefulset + if e := cli.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: statefulset.Name, + }, &statefulset); e != nil { + if apierrs.IsNotFound(e) { + // resource has been successfully deleted + continue + } + // unexpected error, report it + multiErr = multierror.Append(multiErr, e) + } else { + // resource still exists, wait for it to be deleted + return false, nil + } } - return kfDefList, nil + return true, multiErr.ErrorOrNil() } diff --git a/tests/e2e/controller_setup_test.go b/tests/e2e/controller_setup_test.go index 62f2b6e9e3a..bb005142fce 100644 --- a/tests/e2e/controller_setup_test.go +++ b/tests/e2e/controller_setup_test.go @@ -1,4 +1,4 @@ -package e2e +package e2e_test import ( "context" @@ -9,12 +9,10 @@ import ( "time" routev1 "github.com/openshift/api/route/v1" - "github.com/pkg/errors" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" k8sclient "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -49,13 +47,15 @@ type testContext struct { resourceCreationTimeout time.Duration // test DataScienceCluster instance testDsc *dsc.DataScienceCluster + // test DSCI CR because we do not create it in ODH by default + testDSCI *dsci.DSCInitialization // time interval to check for resource creation resourceRetryInterval time.Duration // context for accessing resources ctx context.Context } -func NewTestContext() (*testContext, error) { +func NewTestContext() (*testContext, error) { //nolint:golint,revive // Only used in tests // GetConfig(): If KUBECONFIG env variable is set, it is used to create // the client, else the inClusterConfig() is used. // Lastly if none of them are set, it uses $HOME/.kube/config to create the client. @@ -66,35 +66,31 @@ func NewTestContext() (*testContext, error) { kc, err := k8sclient.NewForConfig(config) if err != nil { - return nil, errors.Wrap(err, "failed to initialize Kubernetes client") + return nil, fmt.Errorf("failed to initialize Kubernetes client: %w", err) } // custom client to manages resources like Route etc custClient, err := client.New(config, client.Options{Scheme: scheme}) if err != nil { - return nil, errors.Wrap(err, "failed to initialize custom client") + return nil, fmt.Errorf("failed to initialize custom client: %w", err) } + // setup DSCI CR since we do not create automatically by operator + testDSCI := setupDSCICR() // Setup DataScienceCluster CR testDSC := setupDSCInstance() - // Get Applications namespace from DSCInitialization instance - dscInit := &dsci.DSCInitialization{} - err = custClient.Get(context.TODO(), types.NamespacedName{Name: "default-dsci"}, dscInit) - if err != nil { - return nil, errors.Wrap(err, "error getting DSCInitialization instance") - } - return &testContext{ cfg: config, kubeClient: kc, customClient: custClient, operatorNamespace: opNamespace, - applicationsNamespace: dscInit.Spec.ApplicationsNamespace, + applicationsNamespace: testDSCI.Spec.ApplicationsNamespace, resourceCreationTimeout: time.Minute * 2, resourceRetryInterval: time.Second * 10, ctx: context.TODO(), testDsc: testDSC, + testDSCI: testDSCI, }, nil } diff --git a/tests/e2e/dsc_cfmap_deletion_test.go b/tests/e2e/dsc_cfmap_deletion_test.go index c106de85a1c..ba887219af8 100644 --- a/tests/e2e/dsc_cfmap_deletion_test.go +++ b/tests/e2e/dsc_cfmap_deletion_test.go @@ -1,11 +1,10 @@ -package e2e +package e2e_test import ( "context" "fmt" "testing" - "github.com/pkg/errors" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -62,7 +61,7 @@ func cfgMapDeletionTestSuite(t *testing.T) { func (tc *testContext) testDSCIDeletion() error { dsciInstances := &dsci.DSCInitializationList{} if err := tc.customClient.List(context.TODO(), dsciInstances); err != nil { - return errors.Wrap(err, "failed while listing DSCIs") + return fmt.Errorf("failed while listing DSCIs: %w", err) } if len(dsciInstances.Items) != 0 { @@ -84,7 +83,7 @@ func (tc *testContext) testDSCDeletionUsingConfigMap() error { if err == nil { dscerr := tc.customClient.Delete(tc.ctx, expectedDSC, &client.DeleteOptions{}) if dscerr != nil { - return fmt.Errorf("error deleting DSC instance %s: %v", expectedDSC.Name, dscerr) + return fmt.Errorf("error deleting DSC instance %s: %w", expectedDSC.Name, dscerr) } } else if !k8serrors.IsNotFound(err) { if err != nil { @@ -96,14 +95,14 @@ func (tc *testContext) testDSCDeletionUsingConfigMap() error { } func (tc *testContext) testOwnedNamespacesDeletion() error { - if err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (done bool, err error) { + if err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) { namespaces, err := tc.kubeClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{ LabelSelector: cluster.ODHGeneratedNamespaceLabel, }) return len(namespaces.Items) == 0, err }); err != nil { - return errors.Wrap(err, "failed waiting for all owned namespaces to be deleted") + return fmt.Errorf("failed waiting for all owned namespaces to be deleted: %w", err) } return nil diff --git a/tests/e2e/dsc_creation_test.go b/tests/e2e/dsc_creation_test.go index 2c6205c1a90..bb5c2a458a1 100644 --- a/tests/e2e/dsc_creation_test.go +++ b/tests/e2e/dsc_creation_test.go @@ -1,4 +1,4 @@ -package e2e +package e2e_test import ( "context" @@ -18,13 +18,22 @@ import ( "k8s.io/client-go/util/retry" dsc "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1" + dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" ) +const ( + odhLabelPrefix = "app.opendatahub.io/" +) + func creationTestSuite(t *testing.T) { testCtx, err := NewTestContext() require.NoError(t, err) t.Run(testCtx.testDsc.Name, func(t *testing.T) { + t.Run("Creation of DSCI CR", func(t *testing.T) { + err = testCtx.testDSCICreation() + require.NoError(t, err, "error creating DSCI CR") + }) t.Run("Creation of DataScienceCluster instance", func(t *testing.T) { err = testCtx.testDSCCreation() require.NoError(t, err, "error creating DataScienceCluster instance") @@ -49,6 +58,43 @@ func creationTestSuite(t *testing.T) { }) } +func (tc *testContext) testDSCICreation() error { + dscLookupKey := types.NamespacedName{Name: tc.testDsc.Name} + createdDSCI := &dsci.DSCInitialization{} + existingDSCIList := &dsci.DSCInitializationList{} + + err := tc.customClient.List(tc.ctx, existingDSCIList) + if err == nil { + // use what you have + if len(existingDSCIList.Items) == 1 { + tc.testDSCI = &existingDSCIList.Items[0] + return nil + } + } + // create one for you + err = tc.customClient.Get(tc.ctx, dscLookupKey, createdDSCI) + if err != nil { + if errors.IsNotFound(err) { + nberr := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) { + creationErr := tc.customClient.Create(tc.ctx, tc.testDSCI) + if creationErr != nil { + log.Printf("error creating DSCI resource %v: %v, trying again", + tc.testDSCI.Name, creationErr) + return false, nil + } + return true, nil + }) + if nberr != nil { + return fmt.Errorf("error creating e2e-test-dsci DSCI CR %s: %w", tc.testDSCI.Name, nberr) + } + } else { + return fmt.Errorf("error getting e2e-test-dsci DSCI CR %s: %w", tc.testDSCI.Name, err) + } + } + + return nil +} + func (tc *testContext) testDSCCreation() error { // Create DataScienceCluster resource if not already created @@ -69,29 +115,28 @@ func (tc *testContext) testDSCCreation() error { err = tc.customClient.Get(tc.ctx, dscLookupKey, createdDSC) if err != nil { if errors.IsNotFound(err) { - nberr := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (done bool, err error) { + nberr := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) { creationErr := tc.customClient.Create(tc.ctx, tc.testDsc) if creationErr != nil { log.Printf("error creating DSC resource %v: %v, trying again", tc.testDsc.Name, creationErr) return false, nil - } else { - return true, nil } + return true, nil }) if nberr != nil { - return fmt.Errorf("error creating e2e-test DSC %s: %v", tc.testDsc.Name, nberr) + return fmt.Errorf("error creating e2e-test DSC %s: %w", tc.testDsc.Name, nberr) } } else { - return fmt.Errorf("error getting e2e-test DSC %s: %v", tc.testDsc.Name, err) + return fmt.Errorf("error getting e2e-test DSC %s: %w", tc.testDsc.Name, err) } } return nil } -func (tc *testContext) testAllApplicationCreation(t *testing.T) error { +func (tc *testContext) testAllApplicationCreation(t *testing.T) error { //nolint:funlen,thelper // Validate test instance is in Ready state dscLookupKey := types.NamespacedName{Name: tc.testDsc.Name} @@ -116,13 +161,9 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { t.Parallel() err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.Dashboard)) if tc.testDsc.Spec.Components.Dashboard.ManagementState == operatorv1.Managed { - if err != nil { - require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.Dashboard.GetComponentName()) - } + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.Dashboard.GetComponentName()) } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Dashboard.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Dashboard.GetComponentName()) } }) @@ -131,13 +172,9 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { t.Parallel() err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.ModelMeshServing)) if tc.testDsc.Spec.Components.ModelMeshServing.ManagementState == operatorv1.Managed { - if err != nil { - require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.ModelMeshServing.GetComponentName()) - } + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.ModelMeshServing.GetComponentName()) } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.ModelMeshServing.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.ModelMeshServing.GetComponentName()) } }) @@ -155,9 +192,7 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { } } } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Kserve.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Kserve.GetComponentName()) } }) @@ -166,13 +201,9 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { t.Parallel() err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.Workbenches)) if tc.testDsc.Spec.Components.Workbenches.ManagementState == operatorv1.Managed { - if err != nil { - require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.Workbenches.GetComponentName()) - } + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.Workbenches.GetComponentName()) } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Workbenches.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Workbenches.GetComponentName()) } }) @@ -181,13 +212,9 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { t.Parallel() err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.DataSciencePipelines)) if tc.testDsc.Spec.Components.DataSciencePipelines.ManagementState == operatorv1.Managed { - if err != nil { - require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.DataSciencePipelines.GetComponentName()) - } + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.DataSciencePipelines.GetComponentName()) } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.DataSciencePipelines.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.DataSciencePipelines.GetComponentName()) } }) @@ -196,16 +223,10 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { t.Parallel() err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.CodeFlare)) if tc.testDsc.Spec.Components.CodeFlare.ManagementState == operatorv1.Managed { - if err != nil { - // dependent operator error, as expected - { - require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.CodeFlare.GetComponentName()) - } - } + // dependent operator error, as expected + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.CodeFlare.GetComponentName()) } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.CodeFlare.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.CodeFlare.GetComponentName()) } }) @@ -214,13 +235,20 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { t.Parallel() err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.Ray)) if tc.testDsc.Spec.Components.Ray.ManagementState == operatorv1.Managed { - if err != nil { - require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.Ray.GetComponentName()) - } + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.Ray.GetComponentName()) } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Ray.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Ray.GetComponentName()) + } + }) + + t.Run("Validate Kueue", func(t *testing.T) { + // speed testing in parallel + t.Parallel() + err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.Kueue)) + if tc.testDsc.Spec.Components.Kueue.ManagementState == operatorv1.Managed { + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.Kueue.GetComponentName()) + } else { + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.Kueue.GetComponentName()) } }) @@ -229,13 +257,20 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { t.Parallel() err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.TrustyAI)) if tc.testDsc.Spec.Components.TrustyAI.ManagementState == operatorv1.Managed { - if err != nil { - require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.TrustyAI.GetComponentName()) - } + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.TrustyAI.GetComponentName()) } else { - if err == nil { - require.NoError(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.TrustyAI.GetComponentName()) - } + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.TrustyAI.GetComponentName()) + } + }) + + t.Run("Validate ModelRegistry", func(t *testing.T) { + // speed testing in parallel + t.Parallel() + err = tc.testApplicationCreation(&(tc.testDsc.Spec.Components.ModelRegistry)) + if tc.testDsc.Spec.Components.ModelRegistry.ManagementState == operatorv1.Managed { + require.NoError(t, err, "error validating application %v when enabled", tc.testDsc.Spec.Components.ModelRegistry.GetComponentName()) + } else { + require.Error(t, err, "error validating application %v when disabled", tc.testDsc.Spec.Components.ModelRegistry.GetComponentName()) } }) @@ -243,15 +278,15 @@ func (tc *testContext) testAllApplicationCreation(t *testing.T) error { } func (tc *testContext) testApplicationCreation(component components.ComponentInterface) error { - err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (done bool, err error) { + err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) { // TODO: see if checking deployment is a good test, CF does not create deployment appList, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: "app.opendatahub.io/" + component.GetComponentName(), + LabelSelector: odhLabelPrefix + component.GetComponentName(), }) if err != nil { log.Printf("error listing application deployments :%v. Trying again...", err) - return false, fmt.Errorf("error listing application deployments :%v. Trying again", err) + return false, fmt.Errorf("error listing application deployments :%w. Trying again", err) } if len(appList.Items) != 0 { allAppDeploymentsReady := true @@ -262,25 +297,19 @@ func (tc *testContext) testApplicationCreation(component components.ComponentInt } if allAppDeploymentsReady { return true, nil - } else { - log.Printf("waiting for application deployments to be in Ready state.") - - return false, nil } - } else { // when no deployment is found - // check Reconcile failed with missing dependent operator error - for _, Condition := range tc.testDsc.Status.Conditions { - if strings.Contains(Condition.Message, "Please install the operator before enabling "+component.GetComponentName()) { - return true, err - } - } - + log.Printf("waiting for application deployments to be in Ready state.") return false, nil } + // when no deployment is found + // check Reconcile failed with missing dependent operator error + for _, Condition := range tc.testDsc.Status.Conditions { + if strings.Contains(Condition.Message, "Please install the operator before enabling "+component.GetComponentName()) { + return true, err + } + } + return false, nil }) - if err != nil { - return err - } return err } @@ -289,7 +318,7 @@ func (tc *testContext) testOwnerrefrences() error { // Test any one of the apps if tc.testDsc.Spec.Components.Dashboard.ManagementState == operatorv1.Managed { appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: "app.opendatahub.io/" + tc.testDsc.Spec.Components.Dashboard.GetComponentName(), + LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(), }) if err != nil { return fmt.Errorf("error listing application deployments %w", err) @@ -308,7 +337,7 @@ func (tc *testContext) testUpdateComponentReconcile() error { // Test Updating Dashboard Replicas appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: "app.opendatahub.io/" + tc.testDsc.Spec.Components.Dashboard.GetComponentName(), + LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(), }) if err != nil { return err @@ -354,7 +383,7 @@ func (tc *testContext) testUpdateDSCComponentEnabled() error { if tc.testDsc.Spec.Components.Dashboard.ManagementState == operatorv1.Managed { appDeployments, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: "app.opendatahub.io/" + tc.testDsc.Spec.Components.Dashboard.GetComponentName(), + LabelSelector: odhLabelPrefix + tc.testDsc.Spec.Components.Dashboard.GetComponentName(), }) if err != nil { return fmt.Errorf("error getting enabled component %v", tc.testDsc.Spec.Components.Dashboard.GetComponentName()) @@ -402,10 +431,9 @@ func (tc *testContext) testUpdateDSCComponentEnabled() error { } return fmt.Errorf("error getting component resource after reconcile: %w", err) - } else { - return fmt.Errorf("component %v is disabled, should not get its deployment %v from NS %v any more", - tc.testDsc.Spec.Components.Dashboard.GetComponentName(), - dashboardDeploymentName, - tc.applicationsNamespace) } + return fmt.Errorf("component %v is disabled, should not get its deployment %v from NS %v any more", + tc.testDsc.Spec.Components.Dashboard.GetComponentName(), + dashboardDeploymentName, + tc.applicationsNamespace) } diff --git a/tests/e2e/dsc_deletion_test.go b/tests/e2e/dsc_deletion_test.go index a3ae74fcb20..94ff3bc159e 100644 --- a/tests/e2e/dsc_deletion_test.go +++ b/tests/e2e/dsc_deletion_test.go @@ -1,4 +1,4 @@ -package e2e +package e2e_test import ( "context" @@ -43,7 +43,7 @@ func (tc *testContext) testDSCDeletion() error { if err == nil { dscerr := tc.customClient.Delete(tc.ctx, expectedDSC, &client.DeleteOptions{}) if dscerr != nil { - return fmt.Errorf("error deleting DSC instance %s: %v", expectedDSC.Name, dscerr) + return fmt.Errorf("error deleting DSC instance %s: %w", expectedDSC.Name, dscerr) } } else if !errors.IsNotFound(err) { if err != nil { @@ -57,9 +57,9 @@ func (tc *testContext) testDSCDeletion() error { func (tc *testContext) testApplicationDeletion(component components.ComponentInterface) error { // Deletion of Deployments - if err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (done bool, err error) { + if err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) { appList, err := tc.kubeClient.AppsV1().Deployments(tc.applicationsNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: "app.opendatahub.io/" + component.GetComponentName(), + LabelSelector: odhLabelPrefix + component.GetComponentName(), }) if err != nil { log.Printf("error listing component deployments :%v. Trying again...", err) @@ -78,37 +78,46 @@ func (tc *testContext) testApplicationDeletion(component components.ComponentInt func (tc *testContext) testAllApplicationDeletion() error { // Deletion all listed components' deployments - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Dashboard)); err != nil { + var err error + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Dashboard)); err != nil { return err } - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.ModelMeshServing)); err != nil { + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.ModelMeshServing)); err != nil { return err } - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Kserve)); err != nil { + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Kserve)); err != nil { return err } - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Workbenches)); err != nil { + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Workbenches)); err != nil { return err } - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.DataSciencePipelines)); err != nil { + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.DataSciencePipelines)); err != nil { return err } - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.CodeFlare)); err != nil { + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.CodeFlare)); err != nil { return err } - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Ray)); err != nil { + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Ray)); err != nil { return err } - if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.TrustyAI)); err != nil { + if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.Kueue)); err != nil { return err } - return nil + if err := tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.TrustyAI)); err != nil { //nolint:revive,nolintlint + return err + } + + if err = tc.testApplicationDeletion(&(tc.testDsc.Spec.Components.ModelRegistry)); err != nil { + return err + } + + return err } diff --git a/tests/e2e/helper_test.go b/tests/e2e/helper_test.go index fec84d9038f..3f9351c3b28 100644 --- a/tests/e2e/helper_test.go +++ b/tests/e2e/helper_test.go @@ -1,4 +1,4 @@ -package e2e +package e2e_test import ( "context" @@ -14,19 +14,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" dsc "github.com/opendatahub-io/opendatahub-operator/v2/apis/datasciencecluster/v1" + dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/components/codeflare" "github.com/opendatahub-io/opendatahub-operator/v2/components/dashboard" "github.com/opendatahub-io/opendatahub-operator/v2/components/datasciencepipelines" "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" + "github.com/opendatahub-io/opendatahub-operator/v2/components/kueue" "github.com/opendatahub-io/opendatahub-operator/v2/components/modelmeshserving" + "github.com/opendatahub-io/opendatahub-operator/v2/components/modelregistry" "github.com/opendatahub-io/opendatahub-operator/v2/components/ray" "github.com/opendatahub-io/opendatahub-operator/v2/components/trustyai" "github.com/opendatahub-io/opendatahub-operator/v2/components/workbenches" ) func (tc *testContext) waitForControllerDeployment(name string, replicas int32) error { - err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (done bool, err error) { + err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) { controllerDeployment, err := tc.kubeClient.AppsV1().Deployments(tc.operatorNamespace).Get(tc.ctx, name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { @@ -53,10 +56,26 @@ func (tc *testContext) waitForControllerDeployment(name string, replicas int32) return err } +func setupDSCICR() *dsci.DSCInitialization { + dsciTest := &dsci.DSCInitialization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e-test-dsci", + }, + Spec: dsci.DSCInitializationSpec{ + ApplicationsNamespace: "opendatahub", + Monitoring: dsci.Monitoring{ + ManagementState: "Managed", + Namespace: "opendatahub", + }, + }, + } + return dsciTest +} + func setupDSCInstance() *dsc.DataScienceCluster { dscTest := &dsc.DataScienceCluster{ ObjectMeta: metav1.ObjectMeta{ - Name: "e2e-test", + Name: "e2e-test-dsc", }, Spec: dsc.DataScienceClusterSpec{ Components: dsc.Components{ @@ -96,11 +115,21 @@ func setupDSCInstance() *dsc.DataScienceCluster { ManagementState: operatorv1.Managed, }, }, + Kueue: kueue.Kueue{ + Component: components.Component{ + ManagementState: operatorv1.Managed, + }, + }, TrustyAI: trustyai.TrustyAI{ Component: components.Component{ ManagementState: operatorv1.Managed, }, }, + ModelRegistry: modelregistry.ModelRegistry{ + Component: components.Component{ + ManagementState: operatorv1.Managed, + }, + }, }, }, } @@ -113,8 +142,8 @@ func (tc *testContext) validateCRD(crdName string) error { obj := client.ObjectKey{ Name: crdName, } - err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (done bool, err error) { - err = tc.customClient.Get(context.TODO(), obj, crd) + err := wait.PollUntilContextTimeout(tc.ctx, tc.resourceRetryInterval, tc.resourceCreationTimeout, false, func(ctx context.Context) (bool, error) { + err := tc.customClient.Get(context.TODO(), obj, crd) if err != nil { if errors.IsNotFound(err) { return false, nil diff --git a/tests/e2e/odh_manager_test.go b/tests/e2e/odh_manager_test.go index ce1aa28227f..4bd39a515ff 100644 --- a/tests/e2e/odh_manager_test.go +++ b/tests/e2e/odh_manager_test.go @@ -1,4 +1,4 @@ -package e2e +package e2e_test import ( "testing" diff --git a/tests/envtestutil/cleaner.go b/tests/envtestutil/cleaner.go index 09eed6430b4..293b02250da 100644 --- a/tests/envtestutil/cleaner.go +++ b/tests/envtestutil/cleaner.go @@ -14,7 +14,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - . "github.com/onsi/gomega" //nolint + . "github.com/onsi/gomega" //nolint:revive,golint,stylecheck // This is the standard for ginkgo and gomega. ) // Cleaner is a struct to perform deletion of resources, @@ -43,6 +43,7 @@ func CreateCleaner(c client.Client, config *rest.Config, timeout, interval time. func (c *Cleaner) DeleteAll(objects ...client.Object) { for _, obj := range objects { + obj := obj Expect(client.IgnoreNotFound(c.client.Delete(context.Background(), obj))).Should(Succeed()) if ns, ok := obj.(*corev1.Namespace); ok { diff --git a/tests/envtestutil/name_gen.go b/tests/envtestutil/name_gen.go index 70507059309..e336a3fd687 100644 --- a/tests/envtestutil/name_gen.go +++ b/tests/envtestutil/name_gen.go @@ -9,11 +9,11 @@ import ( var letters = []rune("abcdefghijklmnopqrstuvwxyz") -func RandomUUIDName(len int) string { - uuidBytes := make([]byte, len) +func RandomUUIDName(numlen int) string { + uuidBytes := make([]byte, numlen) _, _ = rand.Read(uuidBytes) - return hex.EncodeToString(uuidBytes)[:len] + return hex.EncodeToString(uuidBytes)[:numlen] } func AppendRandomNameTo(prefix string) string { diff --git a/tests/integration/features/crd/istio-gateway.crd.yaml b/tests/integration/features/crd/istio-gateway.crd.yaml new file mode 100644 index 00000000000..81d932cd834 --- /dev/null +++ b/tests/integration/features/crd/istio-gateway.crd.yaml @@ -0,0 +1,313 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: gateways.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: Gateway + listKind: GatewayList + plural: gateways + shortNames: + - gw + singular: gateway + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting edge load balancer. See more details + at: https://istio.io/docs/reference/config/networking/gateway.html' + properties: + selector: + additionalProperties: + type: string + description: One or more labels that indicate a specific set of pods/VMs + on which this gateway configuration should be applied. + type: object + servers: + description: A list of server specifications. + items: + properties: + bind: + description: The ip or the Unix domain socket to which the listener + should be bound to. + type: string + defaultEndpoint: + type: string + hosts: + description: One or more hosts exposed by this gateway. + items: + type: string + type: array + name: + description: An optional name of the server, when set must be + unique across all servers. + type: string + port: + description: The Port on which the proxy should listen for incoming + connections. + properties: + name: + description: Label assigned to the port. + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + type: string + targetPort: + type: integer + required: + - number + - protocol + - name + type: object + tls: + description: Set of TLS related options that govern the server's + behavior. + properties: + caCertificates: + description: REQUIRED if mode is `MUTUAL` or `OPTIONAL_MUTUAL`. + type: string + cipherSuites: + description: 'Optional: If specified, only support the specified + cipher list.' + items: + type: string + type: array + credentialName: + description: For gateways running on Kubernetes, the name + of the secret that holds the TLS certs including the CA + certificates. + type: string + httpsRedirect: + description: If set to true, the load balancer will send + a 301 redirect for all http connections, asking the clients + to use HTTPS. + type: boolean + maxProtocolVersion: + description: 'Optional: Maximum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + minProtocolVersion: + description: 'Optional: Minimum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + mode: + description: 'Optional: Indicates whether connections to + this port should be secured using TLS.' + enum: + - PASSTHROUGH + - SIMPLE + - MUTUAL + - AUTO_PASSTHROUGH + - ISTIO_MUTUAL + - OPTIONAL_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + type: string + serverCertificate: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + type: string + subjectAltNames: + description: A list of alternate names to verify the subject + identity in the certificate presented by the client. + items: + type: string + type: array + verifyCertificateHash: + description: An optional list of hex-encoded SHA-256 hashes + of the authorized client certificates. + items: + type: string + type: array + verifyCertificateSpki: + description: An optional list of base64-encoded SHA-256 + hashes of the SPKIs of authorized client certificates. + items: + type: string + type: array + type: object + required: + - port + - hosts + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting edge load balancer. See more details + at: https://istio.io/docs/reference/config/networking/gateway.html' + properties: + selector: + additionalProperties: + type: string + description: One or more labels that indicate a specific set of pods/VMs + on which this gateway configuration should be applied. + type: object + servers: + description: A list of server specifications. + items: + properties: + bind: + description: The ip or the Unix domain socket to which the listener + should be bound to. + type: string + defaultEndpoint: + type: string + hosts: + description: One or more hosts exposed by this gateway. + items: + type: string + type: array + name: + description: An optional name of the server, when set must be + unique across all servers. + type: string + port: + description: The Port on which the proxy should listen for incoming + connections. + properties: + name: + description: Label assigned to the port. + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + type: string + targetPort: + type: integer + required: + - number + - protocol + - name + type: object + tls: + description: Set of TLS related options that govern the server's + behavior. + properties: + caCertificates: + description: REQUIRED if mode is `MUTUAL` or `OPTIONAL_MUTUAL`. + type: string + cipherSuites: + description: 'Optional: If specified, only support the specified + cipher list.' + items: + type: string + type: array + credentialName: + description: For gateways running on Kubernetes, the name + of the secret that holds the TLS certs including the CA + certificates. + type: string + httpsRedirect: + description: If set to true, the load balancer will send + a 301 redirect for all http connections, asking the clients + to use HTTPS. + type: boolean + maxProtocolVersion: + description: 'Optional: Maximum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + minProtocolVersion: + description: 'Optional: Minimum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + mode: + description: 'Optional: Indicates whether connections to + this port should be secured using TLS.' + enum: + - PASSTHROUGH + - SIMPLE + - MUTUAL + - AUTO_PASSTHROUGH + - ISTIO_MUTUAL + - OPTIONAL_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + type: string + serverCertificate: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + type: string + subjectAltNames: + description: A list of alternate names to verify the subject + identity in the certificate presented by the client. + items: + type: string + type: array + verifyCertificateHash: + description: An optional list of hex-encoded SHA-256 hashes + of the authorized client certificates. + items: + type: string + type: array + verifyCertificateSpki: + description: An optional list of base64-encoded SHA-256 + hashes of the SPKIs of authorized client certificates. + items: + type: string + type: array + type: object + required: + - port + - hosts + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} \ No newline at end of file diff --git a/tests/integration/features/crd/openshift-ingresses.yaml b/tests/integration/features/crd/openshift-ingresses.crd.yaml similarity index 100% rename from tests/integration/features/crd/openshift-ingresses.yaml rename to tests/integration/features/crd/openshift-ingresses.crd.yaml diff --git a/tests/integration/features/crd/test-resource.yaml b/tests/integration/features/crd/test-resource.crd.yaml similarity index 100% rename from tests/integration/features/crd/test-resource.yaml rename to tests/integration/features/crd/test-resource.crd.yaml diff --git a/tests/integration/features/features_int_test.go b/tests/integration/features/features_int_test.go index cd8a84e3fcc..3d52595614e 100644 --- a/tests/integration/features/features_int_test.go +++ b/tests/integration/features/features_int_test.go @@ -2,34 +2,47 @@ package features_test import ( "context" + "embed" + "fmt" + "os" + "path" + "path/filepath" "time" + conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" - dscv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" "github.com/opendatahub-io/opendatahub-operator/v2/tests/envtestutil" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" ) +//go:embed templates +var testEmbeddedFiles embed.FS + const ( timeout = 5 * time.Second interval = 250 * time.Millisecond ) -var _ = Describe("preconditions", func() { +var _ = Describe("feature preconditions", func() { Context("namespace existence", func() { var ( objectCleaner *envtestutil.Cleaner - testFeature *feature.Feature namespace string + dsci *dsciv1.DSCInitialization ) BeforeEach(func() { @@ -37,14 +50,7 @@ var _ = Describe("preconditions", func() { testFeatureName := "test-ns-creation" namespace = envtestutil.AppendRandomNameTo(testFeatureName) - - dsciSpec := newDSCInitializationSpec(namespace) - var err error - testFeature, err = feature.CreateFeature(testFeatureName). - For(dsciSpec). - UsingConfig(envTest.Config). - Load() - Expect(err).ToNot(HaveOccurred()) + dsci = newDSCInitialization(namespace) }) It("should create namespace if it does not exist", func() { @@ -54,80 +60,523 @@ var _ = Describe("preconditions", func() { defer objectCleaner.DeleteAll(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}) // when - err = feature.CreateNamespaceIfNotExists(namespace)(testFeature) + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + testFeatureErr := feature.CreateFeature("create-new-ns"). + For(handler). + PreConditions(feature.CreateNamespaceIfNotExists(namespace)). + UsingConfig(envTest.Config). + Load() + + Expect(testFeatureErr).ToNot(HaveOccurred()) + + return nil + }) // then - Expect(err).ToNot(HaveOccurred()) + Expect(featuresHandler.Apply()).To(Succeed()) + + // and + Eventually(func() error { + _, err := getNamespace(namespace) + return err + }).WithTimeout(timeout).WithPolling(interval).Should(Succeed()) }) It("should not try to create namespace if it does already exist", func() { // given - ns := createNamespace(namespace) + ns := newNamespace(namespace) Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + Eventually(func() error { + _, err := getNamespace(namespace) + return err + }).WithTimeout(timeout).WithPolling(interval).Should(Succeed()) // wait for ns to actually get created + defer objectCleaner.DeleteAll(ns) // when - err := feature.CreateNamespaceIfNotExists(namespace)(testFeature) + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + testFeatureErr := feature.CreateFeature("create-new-ns"). + For(handler). + PreConditions(feature.CreateNamespaceIfNotExists(namespace)). + UsingConfig(envTest.Config). + Load() + + Expect(testFeatureErr).ToNot(HaveOccurred()) + + return nil + }) // then - Expect(err).ToNot(HaveOccurred()) + Expect(featuresHandler.Apply()).To(Succeed()) + }) + }) Context("ensuring custom resource definitions are installed", func() { var ( - dsciSpec *dscv1.DSCInitializationSpec - verificationFeature *feature.Feature + dsci *dsciv1.DSCInitialization ) BeforeEach(func() { - dsciSpec = newDSCInitializationSpec("default") + namespace := envtestutil.AppendRandomNameTo("test-crd-creation") + dsci = newDSCInitialization(namespace) }) It("should successfully check for existing CRD", func() { // given example CRD installed into env name := "test-resources.openshift.io" - var err error - verificationFeature, err = feature.CreateFeature("CRD verification"). - For(dsciSpec). - UsingConfig(envTest.Config). - PreConditions(feature.EnsureCRDIsInstalled(name)). - Load() - Expect(err).ToNot(HaveOccurred()) - // when - err = verificationFeature.Apply() + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + crdVerificationErr := feature.CreateFeature("verify-crd-exists"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(feature.EnsureCRDIsInstalled(name)). + Load() + + Expect(crdVerificationErr).ToNot(HaveOccurred()) + + return nil + }) // then - Expect(err).ToNot(HaveOccurred()) + Expect(featuresHandler.Apply()).To(Succeed()) }) It("should fail to check non-existing CRD", func() { // given name := "non-existing-resource.non-existing-group.io" - var err error - verificationFeature, err = feature.CreateFeature("CRD verification"). - For(dsciSpec). - UsingConfig(envTest.Config). - PreConditions(feature.EnsureCRDIsInstalled(name)). - Load() - Expect(err).ToNot(HaveOccurred()) + // when + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + crdVerificationErr := feature.CreateFeature("fail-on-non-existing-crd"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(feature.EnsureCRDIsInstalled(name)). + Load() + + Expect(crdVerificationErr).ToNot(HaveOccurred()) + + return nil + }) + + // then + Expect(featuresHandler.Apply()).To(MatchError(ContainSubstring("\"non-existing-resource.non-existing-group.io\" not found"))) + }) + }) +}) + +var _ = Describe("feature cleanup", func() { + + Context("using FeatureTracker and ownership as cleanup strategy", Ordered, func() { + + const ( + featureName = "create-secret" + secretName = "test-secret" + ) + + var ( + dsci *dsciv1.DSCInitialization + namespace string + featuresHandler *feature.FeaturesHandler + ) + + BeforeAll(func() { + namespace = envtestutil.AppendRandomNameTo("test-secret-ownership") + dsci = newDSCInitialization(namespace) + featuresHandler = feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + secretCreationErr := feature.CreateFeature(featureName). + For(handler). + UsingConfig(envTest.Config). + PreConditions( + feature.CreateNamespaceIfNotExists(namespace), + ). + WithResources(createSecret(secretName, namespace)). + Load() + + Expect(secretCreationErr).ToNot(HaveOccurred()) + return nil + }) + + }) + + It("should successfully create resource and associated feature tracker", func() { // when - err = verificationFeature.Apply() + Expect(featuresHandler.Apply()).Should(Succeed()) // then - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("\"non-existing-resource.non-existing-group.io\" not found")) + Eventually(createdSecretHasOwnerReferenceToOwningFeature(secretName, namespace)). + WithTimeout(timeout). + WithPolling(interval). + Should(Succeed()) + }) + + It("should remove feature tracker on clean-up", func() { + // when + Expect(featuresHandler.Delete()).To(Succeed()) + + // then + Eventually(createdSecretHasOwnerReferenceToOwningFeature(secretName, namespace)). + WithTimeout(timeout). + WithPolling(interval). + Should(WithTransform(errors.IsNotFound, BeTrue())) + }) + + }) + + var _ = Describe("feature trackers", func() { + Context("ensuring feature trackers indicate status and phase", func() { + + const appNamespace = "default" + + var ( + dsci *dsciv1.DSCInitialization + ) + + BeforeEach(func() { + dsci = newDSCInitialization("default") + }) + + It("should indicate successful installation in FeatureTracker", func() { + // given example CRD installed into env + name := "test-resources.openshift.io" + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("crd-verification"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(feature.EnsureCRDIsInstalled(name)). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + featureTracker, err := getFeatureTracker("crd-verification", appNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(*featureTracker.Status.Conditions).To(ContainElement( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(conditionsv1.ConditionAvailable), + "Status": Equal(v1.ConditionTrue), + "Reason": Equal(string(featurev1.FeatureCreated)), + }), + )) + }) + + It("should indicate failure in preconditions", func() { + // given + name := "non-existing-resource.non-existing-group.io" + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("non-existing-crd-verification"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(feature.EnsureCRDIsInstalled(name)). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).ToNot(Succeed()) + + // then + featureTracker, err := getFeatureTracker("non-existing-crd-verification", appNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(*featureTracker.Status.Conditions).To(ContainElement( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(conditionsv1.ConditionDegraded), + "Status": Equal(v1.ConditionTrue), + "Reason": Equal(string(featurev1.PreConditions)), + }), + )) + }) + + It("should indicate failure in post-conditions", func() { + // given + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("post-condition-failure"). + For(handler). + UsingConfig(envTest.Config). + PostConditions(func(f *feature.Feature) error { + return fmt.Errorf("during test always fail") + }). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).ToNot(Succeed()) + + // then + featureTracker, err := getFeatureTracker("post-condition-failure", appNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(*featureTracker.Status.Conditions).To(ContainElement( + MatchFields(IgnoreExtras, Fields{ + "Type": Equal(conditionsv1.ConditionDegraded), + "Status": Equal(v1.ConditionTrue), + "Reason": Equal(string(featurev1.PostConditions)), + }), + )) + }) + + It("should correctly indicate source in the feature tracker", func() { + // given + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + emptyFeatureErr := feature.CreateFeature("empty-feature"). + For(handler). + UsingConfig(envTest.Config). + Load() + + Expect(emptyFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + featureTracker, err := getFeatureTracker("empty-feature", appNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(featureTracker.Spec.Source).To( + MatchFields(IgnoreExtras, Fields{ + "Name": Equal("default-dsci"), + "Type": Equal(featurev1.DSCIType), + }), + ) + }) + + It("should correctly indicate app namespace in the feature tracker", func() { + // given + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + emptyFeatureErr := feature.CreateFeature("empty-feature"). + For(handler). + UsingConfig(envTest.Config). + Load() + + Expect(emptyFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + featureTracker, err := getFeatureTracker("empty-feature", appNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(featureTracker.Spec.AppNamespace).To(Equal("default")) + }) + + }) + + }) + + var _ = Describe("Manifest sources", func() { + Context("using various manifest sources", func() { + + var ( + objectCleaner *envtestutil.Cleaner + dsci *dsciv1.DSCInitialization + namespace *v1.Namespace + ) + + BeforeEach(func() { + objectCleaner = envtestutil.CreateCleaner(envTestClient, envTest.Config, timeout, interval) + nsName := envtestutil.AppendRandomNameTo("smcp-ns") + + var err error + namespace, err = cluster.CreateNamespace(envTestClient, nsName) + Expect(err).ToNot(HaveOccurred()) + + dsci = newDSCInitialization(nsName) + dsci.Spec.ServiceMesh.ControlPlane.Namespace = namespace.Name + }) + + AfterEach(func() { + objectCleaner.DeleteAll(namespace) + }) + + It("should be able to process an embedded template from the default location", func() { + // given + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + createServiceErr := feature.CreateFeature("create-local-gw-svc"). + For(handler). + UsingConfig(envTest.Config). + Manifests(path.Join(feature.ServerlessDir, "serving-istio-gateways", "local-gateway-svc.tmpl")). + Load() + + Expect(createServiceErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + service, err := getService("knative-local-gateway", namespace.Name) + Expect(err).ToNot(HaveOccurred()) + Expect(service.Name).To(Equal("knative-local-gateway")) + }) + + It("should be able to process an embedded YAML file from the default location", func() { + // given + knativeNs, nsErr := cluster.CreateNamespace(envTestClient, "knative-serving") + Expect(nsErr).ToNot(HaveOccurred()) + defer objectCleaner.DeleteAll(knativeNs) + + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + createGatewayErr := feature.CreateFeature("create-local-gateway"). + For(handler). + UsingConfig(envTest.Config). + Manifests(path.Join(feature.ServerlessDir, "serving-istio-gateways", "istio-local-gateway.yaml")). + Load() + + Expect(createGatewayErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + gateway, err := getGateway(envTest.Config, "knative-serving", "knative-local-gateway") + Expect(err).ToNot(HaveOccurred()) + Expect(gateway).ToNot(BeNil()) + }) + + It("should be able to process an embedded file from a non default location", func() { + // given + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + createNamespaceErr := feature.CreateFeature("create-namespace"). + For(handler). + UsingConfig(envTest.Config). + ManifestSource(testEmbeddedFiles). + Manifests(path.Join(feature.BaseDir, "namespace.yaml")). + Load() + + Expect(createNamespaceErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + embeddedNs, err := getNamespace("embedded-test-ns") + defer objectCleaner.DeleteAll(embeddedNs) + Expect(err).ToNot(HaveOccurred()) + Expect(embeddedNs.Name).To(Equal("embedded-test-ns")) + }) + + const nsYAML = `apiVersion: v1 +kind: Namespace +metadata: + name: real-file-test-ns` + + It("should source manifests from a specified temporary directory within the file system", func() { + // given + tempDir := GinkgoT().TempDir() + + Expect(createFile(tempDir, "namespace.yaml", nsYAML)).To(Succeed()) + + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + createServiceErr := feature.CreateFeature("create-namespace"). + For(handler). + UsingConfig(envTest.Config). + ManifestSource(os.DirFS(tempDir)). + Manifests(path.Join("namespace.yaml")). // must be relative to root DirFS defined above + Load() + + Expect(createServiceErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + realNs, err := getNamespace("real-file-test-ns") + defer objectCleaner.DeleteAll(realNs) + Expect(err).ToNot(HaveOccurred()) + Expect(realNs.Name).To(Equal("real-file-test-ns")) + }) }) }) }) -func createNamespace(name string) *v1.Namespace { +func createdSecretHasOwnerReferenceToOwningFeature(secretName, namespace string) func() error { + return func() error { + secret, err := envTestClientset.CoreV1(). + Secrets(namespace). + Get(context.TODO(), secretName, metav1.GetOptions{}) + + if err != nil { + return err + } + + Expect(secret.OwnerReferences).Should( + ContainElement( + MatchFields(IgnoreExtras, Fields{"Kind": Equal("FeatureTracker")}), + ), + ) + + trackerName := "" + for _, ownerRef := range secret.OwnerReferences { + if ownerRef.Kind == "FeatureTracker" { + trackerName = ownerRef.Name + break + } + } + + tracker := &featurev1.FeatureTracker{} + return envTestClient.Get(context.Background(), client.ObjectKey{ + Name: trackerName, + }, tracker) + } +} + +func createSecret(name, namespace string) func(f *feature.Feature) error { + return func(f *feature.Feature) error { + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + f.AsOwnerReference(), + }, + }, + Data: map[string][]byte{ + "test": []byte("test"), + }, + } + + _, err := f.Clientset.CoreV1(). + Secrets(namespace). + Create(context.TODO(), secret, metav1.CreateOptions{}) + + return err + } +} + +func newNamespace(name string) *v1.Namespace { return &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -135,16 +584,52 @@ func createNamespace(name string) *v1.Namespace { } } -func newDSCInitializationSpec(ns string) *dscv1.DSCInitializationSpec { - spec := dscv1.DSCInitializationSpec{} - spec.ApplicationsNamespace = ns +func getFeatureTracker(featureName, appNamespace string) (*featurev1.FeatureTracker, error) { //nolint:unparam //reason appNs + tracker := featurev1.NewFeatureTracker(featureName, appNamespace) + err := envTestClient.Get(context.Background(), client.ObjectKey{ + Name: tracker.Name, + }, tracker) + + return tracker, err +} - return &spec +func newDSCInitialization(ns string) *dsciv1.DSCInitialization { + return &dsciv1.DSCInitialization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default-dsci", + }, + Spec: dsciv1.DSCInitializationSpec{ + ApplicationsNamespace: ns, + }, + } } func getNamespace(namespace string) (*v1.Namespace, error) { - ns := createNamespace(namespace) + ns := newNamespace(namespace) err := envTestClient.Get(context.Background(), types.NamespacedName{Name: namespace}, ns) return ns, err } + +func getService(name, namespace string) (*v1.Service, error) { + svc := &v1.Service{} + err := envTestClient.Get(context.Background(), types.NamespacedName{ + Name: name, Namespace: namespace, + }, svc) + + return svc, err +} + +func createFile(dir, filename, data string) error { + filePath := filepath.Join(dir, filename) + file, err := os.Create(filePath) + if err != nil { + return err + } + + _, err = file.WriteString(data) + if err != nil { + return err + } + return file.Sync() +} diff --git a/tests/integration/features/features_suite_int_test.go b/tests/integration/features/features_suite_int_test.go index dbd1b062821..dd99493c1e9 100644 --- a/tests/integration/features/features_suite_int_test.go +++ b/tests/integration/features/features_suite_int_test.go @@ -11,11 +11,13 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + featurev1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/features/v1" "github.com/opendatahub-io/opendatahub-operator/v2/tests/envtestutil" . "github.com/onsi/ginkgo/v2" @@ -23,10 +25,11 @@ import ( ) var ( - envTestClient client.Client - envTest *envtest.Environment - ctx context.Context - cancel context.CancelFunc + envTestClient client.Client + envTestClientset *kubernetes.Clientset + envTest *envtest.Environment + ctx context.Context + cancel context.CancelFunc ) var testScheme = runtime.NewScheme() @@ -53,6 +56,7 @@ var _ = BeforeSuite(func() { } utilruntime.Must(v1.AddToScheme(testScheme)) + utilruntime.Must(featurev1.AddToScheme(testScheme)) envTest = &envtest.Environment{ CRDInstallOptions: envtest.CRDInstallOptions{ @@ -71,9 +75,16 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(config).NotTo(BeNil()) + err = featurev1.AddToScheme(testScheme) + Expect(err).NotTo(HaveOccurred()) + envTestClient, err = client.New(config, client.Options{Scheme: testScheme}) Expect(err).NotTo(HaveOccurred()) Expect(envTestClient).NotTo(BeNil()) + + envTestClientset, err = kubernetes.NewForConfig(config) + Expect(err).NotTo(HaveOccurred()) + Expect(envTestClientset).NotTo(BeNil()) }) var _ = AfterSuite(func() { diff --git a/tests/integration/features/serverless_feature_test.go b/tests/integration/features/serverless_feature_test.go index 26d7b4192f1..9aa18693065 100644 --- a/tests/integration/features/serverless_feature_test.go +++ b/tests/integration/features/serverless_feature_test.go @@ -2,14 +2,19 @@ package features_test import ( "context" + "fmt" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/dynamic" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/components/kserve" infrav1 "github.com/opendatahub-io/opendatahub-operator/v2/infrastructure/v1" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/serverless" @@ -64,45 +69,59 @@ spec: var _ = Describe("Serverless feature", func() { - var testFeature *feature.Feature - var objectCleaner *envtestutil.Cleaner + var ( + dsci *dsciv1.DSCInitialization + objectCleaner *envtestutil.Cleaner + kserveComponent *kserve.Kserve + ) BeforeEach(func() { + // TODO rework c, err := client.New(envTest.Config, client.Options{}) Expect(err).ToNot(HaveOccurred()) - objectCleaner = envtestutil.CreateCleaner(c, envTest.Config, timeout, interval) - testFeatureName := "serverless-feature" - namespace := envtestutil.AppendRandomNameTo(testFeatureName) - - dsciSpec := newDSCInitializationSpec(namespace) - testFeature, err = feature.CreateFeature(testFeatureName). - For(dsciSpec). - UsingConfig(envTest.Config). - Load() - - Expect(err).ToNot(HaveOccurred()) + dsci = newDSCInitialization("default") + kserveComponent = &kserve.Kserve{} }) Context("verifying preconditions", func() { When("operator is not installed", func() { - It("operator presence check should return an error", func() { - Expect(serverless.EnsureServerlessOperatorInstalled(testFeature)).To(HaveOccurred()) + + It("should fail on precondition check", func() { + // given + featuresHandler := feature.ComponentFeaturesHandler(kserveComponent, &dsci.Spec, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("no-serverless-operator-check"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(serverless.EnsureServerlessOperatorInstalled). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + applyErr := featuresHandler.Apply() + + // then + Expect(applyErr).To(MatchError(ContainSubstring("\"knativeservings.operator.knative.dev\" not found"))) }) }) When("operator is installed", func() { + var knativeServingCrdObj *apiextensionsv1.CustomResourceDefinition BeforeEach(func() { // Create KNativeServing the CRD knativeServingCrdObj = &apiextensionsv1.CustomResourceDefinition{} - Expect(yaml.Unmarshal([]byte(knativeServingCrd), knativeServingCrdObj)).ToNot(HaveOccurred()) + Expect(yaml.Unmarshal([]byte(knativeServingCrd), knativeServingCrdObj)).To(Succeed()) c, err := client.New(envTest.Config, client.Options{}) Expect(err).ToNot(HaveOccurred()) - Expect(c.Create(context.TODO(), knativeServingCrdObj)).ToNot(HaveOccurred()) + Expect(c.Create(context.TODO(), knativeServingCrdObj)).To(Succeed()) crdOptions := envtest.CRDInstallOptions{PollInterval: interval, MaxTime: timeout} err = envtest.WaitForCRDs(envTest.Config, []*apiextensionsv1.CustomResourceDefinition{knativeServingCrdObj}, crdOptions) @@ -114,32 +133,95 @@ var _ = Describe("Serverless feature", func() { objectCleaner.DeleteAll(knativeServingCrdObj) }) - It("operator presence check should succeed", func() { - Expect(serverless.EnsureServerlessOperatorInstalled(testFeature)).ToNot(HaveOccurred()) + It("should succeed checking operator installation using precondition", func() { + // when + featuresHandler := feature.ComponentFeaturesHandler(kserveComponent, &dsci.Spec, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("serverless-operator-check"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(serverless.EnsureServerlessOperatorInstalled). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // then + Expect(featuresHandler.Apply()).To(Succeed()) }) - It("KNative serving absence check should succeed if serving is not installed", func() { - Expect(serverless.EnsureServerlessAbsent(testFeature)).ToNot(HaveOccurred()) + It("should succeed if serving is not installed for KNative serving precondition", func() { + // when + featuresHandler := feature.ComponentFeaturesHandler(kserveComponent, &dsci.Spec, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("no-serving-installed-yet"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(serverless.EnsureServerlessAbsent). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // then + Expect(featuresHandler.Apply()).To(Succeed()) }) - It("KNative serving absence check should fail when serving is present", func() { + It("should fail if serving is already installed for KNative serving precondition", func() { + // given ns := envtestutil.AppendRandomNameTo(testNamespacePrefix) - nsResource := createNamespace(ns) - Expect(envTestClient.Create(context.TODO(), nsResource)).ToNot(HaveOccurred()) + nsResource := newNamespace(ns) + Expect(envTestClient.Create(context.TODO(), nsResource)).To(Succeed()) defer objectCleaner.DeleteAll(nsResource) knativeServing := &unstructured.Unstructured{} - Expect(yaml.Unmarshal([]byte(knativeServingInstance), knativeServing)).ToNot(HaveOccurred()) + Expect(yaml.Unmarshal([]byte(knativeServingInstance), knativeServing)).To(Succeed()) knativeServing.SetNamespace(nsResource.Name) - Expect(envTestClient.Create(context.TODO(), knativeServing)).ToNot(HaveOccurred()) + Expect(envTestClient.Create(context.TODO(), knativeServing)).To(Succeed()) + + // when + featuresHandler := feature.ComponentFeaturesHandler(kserveComponent, &dsci.Spec, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("serving-already-installed"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(serverless.EnsureServerlessAbsent). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) - Expect(serverless.EnsureServerlessAbsent(testFeature)).To(HaveOccurred()) + // then + Expect(featuresHandler.Apply()).ToNot(Succeed()) }) }) + }) Context("default values", func() { + var testFeature *feature.Feature + + BeforeEach(func() { + // Stubbing feature as we want to test particular functions in isolation + testFeature = &feature.Feature{ + Name: "test-feature", + Spec: &feature.Spec{ + ServiceMeshSpec: &infrav1.ServiceMeshSpec{}, + Serving: &infrav1.ServingSpec{}, + }, + } + + testFeature.Client = envTestClient + testFeature.Clientset = envTestClientset + var err error + testFeature.DynamicClient, err = dynamic.NewForConfig(envTest.Config) + Expect(err).ToNot(HaveOccurred()) + }) + Context("ingress gateway TLS secret name", func() { It("should set default value when value is empty in the DSCI", func() { @@ -163,16 +245,16 @@ var _ = Describe("Serverless feature", func() { Expect(yaml.Unmarshal([]byte(openshiftClusterIngress), osIngressResource)).ToNot(HaveOccurred()) c, err := client.New(envTest.Config, client.Options{}) Expect(err).ToNot(HaveOccurred()) - Expect(c.Create(context.TODO(), osIngressResource)).ToNot(HaveOccurred()) + Expect(c.Create(context.TODO(), osIngressResource)).To(Succeed()) // Default value is blank -> testFeature.Spec.Serving.IngressGateway.Domain = "" - Expect(serverless.ServingIngressDomain(testFeature)).ToNot(HaveOccurred()) + Expect(serverless.ServingIngressDomain(testFeature)).To(Succeed()) Expect(testFeature.Spec.KnativeIngressDomain).To(Equal("*.foo.io")) }) It("should use user value when set in the DSCI", func() { testFeature.Spec.Serving.IngressGateway.Domain = testDomainFooCom - Expect(serverless.ServingIngressDomain(testFeature)).ToNot(HaveOccurred()) + Expect(serverless.ServingIngressDomain(testFeature)).To(Succeed()) Expect(testFeature.Spec.KnativeIngressDomain).To(Equal(testDomainFooCom)) }) }) @@ -180,43 +262,97 @@ var _ = Describe("Serverless feature", func() { Context("resources creation", func() { - It("should create a TLS secret if certificate is SelfSigned", func() { + var ( + namespace *corev1.Namespace + ) + + BeforeEach(func() { ns := envtestutil.AppendRandomNameTo(testNamespacePrefix) - nsResource := createNamespace(ns) - Expect(envTestClient.Create(context.TODO(), nsResource)).ToNot(HaveOccurred()) - defer objectCleaner.DeleteAll(nsResource) + namespace = newNamespace(ns) + Expect(envTestClient.Create(context.TODO(), namespace)).To(Succeed()) - testFeature.Spec.ControlPlane.Namespace = nsResource.Name - testFeature.Spec.Serving.IngressGateway.Certificate.Type = infrav1.SelfSigned - testFeature.Spec.Serving.IngressGateway.Domain = testDomainFooCom - Expect(serverless.ServingDefaultValues(testFeature)).ToNot(HaveOccurred()) - Expect(serverless.ServingIngressDomain(testFeature)).ToNot(HaveOccurred()) + dsci.Spec.ServiceMesh.ControlPlane.Namespace = ns + }) - Expect(serverless.ServingCertificateResource(testFeature)).ToNot(HaveOccurred()) + AfterEach(func() { + objectCleaner.DeleteAll(namespace) + }) - secret, err := testFeature.Clientset.CoreV1().Secrets(nsResource.Name).Get(context.TODO(), serverless.DefaultCertificateSecretName, v1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - Expect(secret).ToNot(BeNil()) + It("should create a TLS secret if certificate is SelfSigned", func() { + // given + kserveComponent.Serving.IngressGateway.Certificate.Type = infrav1.SelfSigned + kserveComponent.Serving.IngressGateway.Domain = testDomainFooCom + + featuresHandler := feature.ComponentFeaturesHandler(kserveComponent, &dsci.Spec, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("tls-secret-creation"). + For(handler). + UsingConfig(envTest.Config). + WithData( + kserve.PopulateComponentSettings(kserveComponent), + serverless.ServingDefaultValues, + serverless.ServingIngressDomain, + ). + WithResources(serverless.ServingCertificateResource). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + + // then + Eventually(func() error { + secret, err := envTestClientset.CoreV1().Secrets(namespace.Name).Get(context.TODO(), serverless.DefaultCertificateSecretName, metav1.GetOptions{}) + if err != nil { + return err + } + + if secret == nil { + return fmt.Errorf("secret not found") + } + + return nil + }).WithTimeout(timeout).WithPolling(interval).Should(Succeed()) }) It("should not create any TLS secret if certificate is user provided", func() { - ns := envtestutil.AppendRandomNameTo(testNamespacePrefix) - nsResource := createNamespace(ns) - Expect(envTestClient.Create(context.TODO(), nsResource)).ToNot(HaveOccurred()) - defer objectCleaner.DeleteAll(nsResource) + // given + kserveComponent.Serving.IngressGateway.Certificate.Type = infrav1.Provided + kserveComponent.Serving.IngressGateway.Domain = testDomainFooCom + featuresHandler := feature.ComponentFeaturesHandler(kserveComponent, &dsci.Spec, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("tls-secret-creation"). + For(handler). + UsingConfig(envTest.Config). + WithData( + kserve.PopulateComponentSettings(kserveComponent), + serverless.ServingDefaultValues, + serverless.ServingIngressDomain, + ). + WithResources(serverless.ServingCertificateResource). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) - testFeature.Spec.ControlPlane.Namespace = nsResource.Name - testFeature.Spec.Serving.IngressGateway.Certificate.Type = infrav1.Provided - testFeature.Spec.Serving.IngressGateway.Domain = "*.foo.com" - Expect(serverless.ServingDefaultValues(testFeature)).ToNot(HaveOccurred()) - Expect(serverless.ServingIngressDomain(testFeature)).ToNot(HaveOccurred()) + // when + Expect(featuresHandler.Apply()).To(Succeed()) - Expect(serverless.ServingCertificateResource(testFeature)).ToNot(HaveOccurred()) + // then + Consistently(func() error { + list, err := envTestClientset.CoreV1().Secrets(namespace.Name).List(context.TODO(), metav1.ListOptions{}) + if err != nil || len(list.Items) != 0 { + return fmt.Errorf("list len: %d, error: %w", len(list.Items), err) + } - list, err := testFeature.Clientset.CoreV1().Secrets(nsResource.Name).List(context.TODO(), v1.ListOptions{}) - Expect(err).ToNot(HaveOccurred()) - Expect(list.Items).To(BeEmpty()) + return nil + }).WithTimeout(timeout).WithPolling(interval).Should(Succeed()) }) }) + }) diff --git a/tests/integration/features/servicemesh_feature_test.go b/tests/integration/features/servicemesh_feature_test.go index b05bf8cd72a..bf32225611c 100644 --- a/tests/integration/features/servicemesh_feature_test.go +++ b/tests/integration/features/servicemesh_feature_test.go @@ -2,17 +2,18 @@ package features_test import ( "context" - "fmt" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + dsciv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" @@ -22,7 +23,7 @@ import ( . "github.com/onsi/gomega" ) -const smcpCrd = `apiVersion: apiextensions.k8s.io/v1 +const serviceMeshControlPlaneCRD = `apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: labels: @@ -65,40 +66,61 @@ spec: ` var _ = Describe("Service Mesh feature", func() { - var testFeature *feature.Feature - var objectCleaner *envtestutil.Cleaner + + var ( + dsci *dsciv1.DSCInitialization + objectCleaner *envtestutil.Cleaner + ) BeforeEach(func() { c, err := client.New(envTest.Config, client.Options{}) Expect(err).ToNot(HaveOccurred()) - objectCleaner = envtestutil.CreateCleaner(c, envTest.Config, timeout, interval) - testFeatureName := "servicemesh-feature" - namespace := envtestutil.AppendRandomNameTo(testFeatureName) + namespace := envtestutil.AppendRandomNameTo("service-mesh-settings") - dsciSpec := newDSCInitializationSpec(namespace) - testFeature, err = feature.CreateFeature(testFeatureName). - For(dsciSpec). - UsingConfig(envTest.Config). - Load() + dsci = newDSCInitialization(namespace) Expect(err).ToNot(HaveOccurred()) }) + AfterEach(func() { + + }) + Describe("preconditions", func() { + When("operator is not installed", func() { - It("operator presence check should return an error", func() { - Expect(servicemesh.EnsureServiceMeshOperatorInstalled(testFeature)).To(HaveOccurred()) + + It("should fail using precondition check", func() { + // given + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("no-serverless-operator-check"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(servicemesh.EnsureServiceMeshOperatorInstalled). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + applyErr := featuresHandler.Apply() + + // then + Expect(applyErr).To(MatchError(ContainSubstring("customresourcedefinitions.apiextensions.k8s.io \"servicemeshcontrolplanes.maistra.io\" not found"))) }) }) + When("operator is installed", func() { var smcpCrdObj *apiextensionsv1.CustomResourceDefinition BeforeEach(func() { // Create SMCP the CRD smcpCrdObj = &apiextensionsv1.CustomResourceDefinition{} - Expect(yaml.Unmarshal([]byte(smcpCrd), smcpCrdObj)).ToNot(HaveOccurred()) + Expect(yaml.Unmarshal([]byte(serviceMeshControlPlaneCRD), smcpCrdObj)).ToNot(HaveOccurred()) c, err := client.New(envTest.Config, client.Options{}) Expect(err).ToNot(HaveOccurred()) Expect(c.Create(context.TODO(), smcpCrdObj)).ToNot(HaveOccurred()) @@ -107,35 +129,106 @@ var _ = Describe("Service Mesh feature", func() { err = envtest.WaitForCRDs(envTest.Config, []*apiextensionsv1.CustomResourceDefinition{smcpCrdObj}, crdOptions) Expect(err).ToNot(HaveOccurred()) }) + AfterEach(func() { // Delete SMCP CRD objectCleaner.DeleteAll(smcpCrdObj) }) - It("operator presence check should succeed", func() { - Expect(servicemesh.EnsureServiceMeshOperatorInstalled(testFeature)).To(Succeed()) + + It("should succeed using precondition check", func() { + // when + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("service-mesh-operator-check"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(servicemesh.EnsureServiceMeshOperatorInstalled). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // when + Expect(featuresHandler.Apply()).To(Succeed()) + }) + It("should find installed Service Mesh Control Plane", func() { + // given c, err := client.New(envTest.Config, client.Options{}) Expect(err).ToNot(HaveOccurred()) ns := envtestutil.AppendRandomNameTo(testNamespacePrefix) - nsResource := createNamespace(ns) + nsResource := newNamespace(ns) Expect(c.Create(context.Background(), nsResource)).To(Succeed()) defer objectCleaner.DeleteAll(nsResource) createServiceMeshControlPlane("test-name", ns) + dsci.Spec.ServiceMesh.ControlPlane.Namespace = ns + dsci.Spec.ServiceMesh.ControlPlane.Name = "test-name" - testFeature.Spec.ControlPlane.Namespace = ns - testFeature.Spec.ControlPlane.Name = "test-name" - Expect(servicemesh.EnsureServiceMeshInstalled(testFeature)).To(Succeed()) + // when + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("service-mesh-control-plane-check"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(servicemesh.EnsureServiceMeshInstalled). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // then + Expect(featuresHandler.Apply()).To(Succeed()) }) + It("should fail to find Service Mesh Control Plane if not present", func() { - Expect(servicemesh.EnsureServiceMeshInstalled(testFeature)).ToNot(Succeed()) + // given + dsci.Spec.ServiceMesh.ControlPlane.Name = "test-name" + + // when + featuresHandler := feature.ClusterFeaturesHandler(dsci, func(handler *feature.FeaturesHandler) error { + verificationFeatureErr := feature.CreateFeature("no-service-mesh-control-plane-check"). + For(handler). + UsingConfig(envTest.Config). + PreConditions(servicemesh.EnsureServiceMeshInstalled). + Load() + + Expect(verificationFeatureErr).ToNot(HaveOccurred()) + + return nil + }) + + // then + Expect(featuresHandler.Apply()).To(MatchError(ContainSubstring("failed to find Service Mesh Control Plane"))) }) + }) + }) }) +func getGateway(cfg *rest.Config, namespace, name string) (*unstructured.Unstructured, error) { + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + gwGvr := schema.GroupVersionResource{ + Group: "networking.istio.io", + Version: "v1beta1", + Resource: "gateways", + } + + gateway, err := dynamicClient.Resource(gwGvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return gateway, nil +} + func createServiceMeshControlPlane(name, namespace string) { serviceMeshControlPlane := &unstructured.Unstructured{ Object: map[string]interface{}{ @@ -151,7 +244,7 @@ func createServiceMeshControlPlane(name, namespace string) { Expect(createSMCPInCluster(envTest.Config, serviceMeshControlPlane, namespace)).To(Succeed()) } -// createSMCPInCluster uses dynamic client to create a dummy SMCP resource for testing +// createSMCPInCluster uses dynamic client to create a dummy SMCP resource for testing. func createSMCPInCluster(cfg *rest.Config, smcpObj *unstructured.Unstructured, namespace string) error { dynamicClient, err := dynamic.NewForConfig(cfg) if err != nil { @@ -189,11 +282,10 @@ func createSMCPInCluster(cfg *rest.Config, smcpObj *unstructured.Unstructured, n return err } - r, err := dynamicClient.Resource(gvr.SMCP).Namespace(namespace).UpdateStatus(context.TODO(), result, metav1.UpdateOptions{}) + _, err = dynamicClient.Resource(gvr.SMCP).Namespace(namespace).UpdateStatus(context.TODO(), result, metav1.UpdateOptions{}) if err != nil { return err } - fmt.Printf("result: %v", r) return nil } diff --git a/tests/integration/features/templates/namespace.yaml b/tests/integration/features/templates/namespace.yaml new file mode 100644 index 00000000000..6e6e5ab44cb --- /dev/null +++ b/tests/integration/features/templates/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: embedded-test-ns