local operator catalog raw file based format (#107)

* local operator catalog raw file based format

* undo local testing setting

* multiple subscriptions not needed, only one

* Makefile refactor

* doc/development.md: catalog doc

* github actions for the catalog image

* lint issues fixed

* Update doc/development.md

Co-authored-by: dd di cesare <4183971+didierofrivia@users.noreply.github.com>

Co-authored-by: Alex Snaps <alex@wcgw.dev>
Co-authored-by: dd di cesare <4183971+didierofrivia@users.noreply.github.com>

.github/workflows/build-images.yaml

@@ -2,8 +2,8 @@ name: Build Images
 
 on:
   push:
-    branches: [ '*' ]
-    tags: [ '*' ]
+    branches: ['*']
+    tags: ['*']
 
 env:
   IMG_TAGS: ${{ github.ref_name }}
@@ -69,9 +69,6 @@ jobs:
       - name: Run make bundle
         if: ${{ github.ref_name != env.MAIN_BRANCH_NAME }}
         run: make bundle REGISTRY=${{ env.IMG_REGISTRY_HOST }} ORG=${{ env.IMG_REGISTRY_ORG }} IMAGE_TAG=${{ github.ref_name }}
-      - name: Run make bundle (main)
-        if: ${{ github.ref_name == env.MAIN_BRANCH_NAME }}
-        run: make bundle REGISTRY=${{ env.IMG_REGISTRY_HOST }} ORG=${{ env.IMG_REGISTRY_ORG }} IMAGE_TAG=latest VERSION=0.0.0
       - name: Git diff
         run: git diff
       - name: Install qemu dependency
@@ -117,14 +114,8 @@ jobs:
         id: add-latest-tag
         run: |
           echo "IMG_TAGS=latest ${{ env.IMG_TAGS }}" >> $GITHUB_ENV
-      - name: Run make catalog-generate
-        if: ${{ github.ref_name != env.MAIN_BRANCH_NAME }}
-        run: make catalog-generate REGISTRY=${{ env.IMG_REGISTRY_HOST }} ORG=${{ env.IMG_REGISTRY_ORG }} IMAGE_TAG=${{ github.ref_name }}
-      - name: Run make catalog-generate (main)
-        if: ${{ github.ref_name == env.MAIN_BRANCH_NAME }}
-        run: make catalog-generate REGISTRY=${{ env.IMG_REGISTRY_HOST }} ORG=${{ env.IMG_REGISTRY_ORG }} IMAGE_TAG=latest VERSION=0.0.0
-      - name: Git diff
-        run: git diff
+      - name: Generate Catalog Content
+        run: make catalog
       - name: Install qemu dependency
         run: |
           sudo apt-get update
@@ -136,8 +127,9 @@ jobs:
           image: kuadrant-operator-catalog
           tags: ${{ env.IMG_TAGS }}
           platforms: linux/amd64,linux/arm64
+          context: ./catalog
           dockerfiles: |
-            ./index.Dockerfile
+            ./catalog/kuadrant-operator-catalog.Dockerfile
       - name: Push Image
         if: ${{ !env.ACT }}
         id: push-to-quay

.gitignore

@@ -26,6 +26,5 @@ testbin/*
 *~
 kuadrant-operator
 
-# ./bin/opm index add --generate output
-database/index.db
-index.Dockerfile
+/catalog/kuadrant-operator-catalog
+/catalog/kuadrant-operator-catalog.Dockerfile

Makefile

@@ -78,14 +78,6 @@ SHELL = /usr/bin/env bash -o pipefail
 KUADRANT_NAMESPACE ?= kuadrant-system
 
 # Kuadrant component versions
-## kuadrant controller
-#ToDo Pin this version once we have an initial release of the controller
-KUADRANT_CONTROLLER_VERSION ?= latest
-ifeq (latest,$(KUADRANT_CONTROLLER_VERSION))
-KUADRANT_CONTROLLER_GITREF = main
-else
-KUADRANT_CONTROLLER_GITREF = $(KUADRANT_CONTROLLER_VERSION)
-endif
 ## authorino
 #ToDo Pin this version once we have an initial release of authorino
 AUTHORINO_OPERATOR_VERSION ?= latest
@@ -94,7 +86,7 @@ AUTHORINO_OPERATOR_BUNDLE_VERSION = 0.0.0
 AUTHORINO_OPERATOR_BUNDLE_IMG_TAG = latest
 AUTHORINO_OPERATOR_GITREF = main
 else
-AUTHORINO_OPERATOR_BUNDLE_VERSION = ${AUTHORINO_OPERATOR_VERSION}
+AUTHORINO_OPERATOR_BUNDLE_VERSION = $(AUTHORINO_OPERATOR_VERSION)
 AUTHORINO_OPERATOR_BUNDLE_IMG_TAG = v$(AUTHORINO_OPERATOR_BUNDLE_VERSION)
 AUTHORINO_OPERATOR_GITREF = v$(AUTHORINO_OPERATOR_BUNDLE_VERSION)
 endif
@@ -131,9 +123,69 @@ all: build
 help: ## Display this help.
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
+##@ Tools
+
+OPERATOR_SDK = $(PROJECT_PATH)/bin/operator-sdk
+OPERATOR_SDK_VERSION = v1.22.0
+$(OPERATOR_SDK):
+	./utils/install-operator-sdk.sh $(OPERATOR_SDK) $(OPERATOR_SDK_VERSION)
+
+.PHONY: operator-sdk
+operator-sdk: $(OPERATOR_SDK) ## Download operator-sdk locally if necessary.
+
+CONTROLLER_GEN = $(PROJECT_PATH)/bin/controller-gen
+$(CONTROLLER_GEN):
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.10.0)
+
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN)  ## Download controller-gen locally if necessary.
+
+KUSTOMIZE = $(PROJECT_PATH)/bin/kustomize
+$(KUSTOMIZE):
+	$(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.5)
+
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
+
+YQ=$(PROJECT_PATH)/bin/yq
+$(YQ):
+	$(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4@latest)
+
+.PHONY: yq
+yq: $(YQ) ## Download yq locally if necessary.
+
+OPM = $(PROJECT_PATH)/bin/opm
+OPM_VERSION = v1.26.2
+$(OPM):
+	@{ \
+	set -e ;\
+	mkdir -p $(dir $(OPM)) ;\
+	OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
+	curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/$(OPM_VERSION)/$${OS}-$${ARCH}-opm ;\
+	chmod +x $(OPM) ;\
+	}
+
+.PHONY: opm
+opm: $(OPM) ## Download opm locally if necessary.
+
+KIND = $(PROJECT_PATH)/bin/kind
+$(KIND):
+	$(call go-install-tool,$(KIND),sigs.k8s.io/kind@v0.11.1)
+
+.PHONY: kind
+kind: $(KIND) ## Download kind locally if necessary.
+
+ACT = $(PROJECT_PATH)/bin/act
+$(ACT):
+	$(call go-install-tool,$(ACT),github.com/nektos/act@latest)
+
+.PHONY: act
+act: $(ACT) ## Download act locally if necessary.
+
 ##@ Development
 
-manifests: controller-gen dependencies-manifests ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+.PHONY: manifests
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: dependencies-manifests
@@ -147,6 +199,7 @@ dependencies-manifests: ## Update kuadrant dependencies manifests.
         < config/dependencies/limitador/kustomization.template.yaml \
         > config/dependencies/limitador/kustomization.yaml
 
+.PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
@@ -174,12 +227,11 @@ namespace: ## Creates a namespace where to deploy Kuadrant Operator
 	kubectl create namespace $(KUADRANT_NAMESPACE)
 
 .PHONY: local-setup
-local-setup: export IMG := $(IMAGE_TAG_BASE):dev
-local-setup: ## Deploy locally kuadrant operator from the current code
+local-setup: $(KIND) ## Deploy locally kuadrant operator from the current code
 	$(MAKE) local-env-setup
-	$(MAKE) docker-build
-	$(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME)
-	$(MAKE) deploy
+	$(MAKE) docker-build IMG=$(IMAGE_TAG_BASE):dev
+	$(KIND) load docker-image $(IMAGE_TAG_BASE):dev --name $(KIND_CLUSTER_NAME)
+	$(MAKE) deploy IMG=$(IMAGE_TAG_BASE):dev
 	kubectl -n $(KUADRANT_NAMESPACE) wait --timeout=300s --for=condition=Available deployments --all
 	@echo
 	@echo "Now you can export the kuadrant gateway by doing:"
@@ -225,10 +277,10 @@ run: generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
 docker-build: ## Build docker image with the manager.
-	docker build -t ${IMG} .
+	docker build -t $(IMG) .
 
 docker-push: ## Push docker image with the manager.
-	docker push ${IMG}
+	docker push $(IMG)
 
 ##@ Deployment
 
@@ -238,10 +290,10 @@ install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/crd | kubectl delete -f -
 
-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+deploy: manifests dependencies-manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
 	$(KUSTOMIZE) build config/deploy | kubectl apply -f -
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMAGE_TAG_BASE}:latest
+	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE_TAG_BASE):latest
 
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/deploy | kubectl delete -f -
@@ -251,34 +303,28 @@ deploy-dependencies: kustomize dependencies-manifests ## Deploy dependencies to
 	kubectl -n "$(KUADRANT_NAMESPACE)" wait --timeout=300s --for=condition=Available deployments --all
 
 .PHONY: install-olm
-install-olm:
+install-olm: $(OPERATOR_SDK)
 	$(OPERATOR_SDK) olm install
 
 .PHONY: uninstall-olm
 uninstall-olm:
 	$(OPERATOR_SDK) olm uninstall
 
-deploy-olm: ## Deploy controller to the K8s cluster specified in ~/.kube/config using OLM catalog image.
+deploy-catalog: $(KUSTOMIZE) $(YQ) ## Deploy controller to the K8s cluster specified in ~/.kube/config using OLM catalog image.
+	V="$(CATALOG_IMG)" $(YQ) eval '.spec.image = strenv(V)' -i config/deploy/olm/catalogsource.yaml
 	$(KUSTOMIZE) build config/deploy/olm | kubectl apply -f -
 
-undeploy-olm: ## Undeploy controller from the K8s cluster specified in ~/.kube/config using OLM catalog image.
+undeploy-catalog: $(KUSTOMIZE) ## Undeploy controller from the K8s cluster specified in ~/.kube/config using OLM catalog image.
 	$(KUSTOMIZE) build config/deploy/olm | kubectl delete -f -
 
-CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
-controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.10.0)
-
-KUSTOMIZE = $(shell pwd)/bin/kustomize
-kustomize: ## Download kustomize locally if necessary.
-	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.5)
 
 ENVTEST = $(shell pwd)/bin/setup-envtest
 envtest: ## Download envtest-setup locally if necessary.
-	$(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
+	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
 
-# go-get-tool will 'go install' any package $2 and install it to $1.
+# go-install-tool will 'go install' any package $2 and install it to $1.
 PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
-define go-get-tool
+define go-install-tool
 @[ -f $(1) ] || { \
 set -e ;\
 TMP_DIR=$$(mktemp -d) ;\
@@ -290,27 +336,24 @@ rm -rf $$TMP_DIR ;\
 }
 endef
 
-OPERATOR_SDK = $(shell pwd)/bin/operator-sdk
-OPERATOR_SDK_VERSION = v1.22.0
-operator-sdk: ## Download operator-sdk locally if necessary.
-	./utils/install-operator-sdk.sh $(OPERATOR_SDK) $(OPERATOR_SDK_VERSION)
-
-.PHONY: bundle-dependencies
-bundle-dependencies: export AUTHORINO_OPERATOR_BUNDLE_VERSION := $(AUTHORINO_OPERATOR_BUNDLE_VERSION)
-bundle-dependencies: export LIMITADOR_OPERATOR_BUNDLE_VERSION := $(LIMITADOR_OPERATOR_BUNDLE_VERSION)
-bundle-dependencies: ## Generate bundle dependencies file.
-	./utils/generate-dependencies-yaml.sh > bundle/metadata/dependencies.yaml
-
 .PHONY: bundle
-bundle: export IMAGE_TAG := $(IMAGE_TAG)
-bundle: export BUNDLE_VERSION := $(VERSION)
-bundle: manifests kustomize operator-sdk bundle-dependencies ## Generate bundle manifests and metadata, then validate generated files.
+bundle: $(OPM) $(YQ) manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files.
 	$(OPERATOR_SDK) generate kustomize manifests -q
+	# Set desired operator image
 	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
-	envsubst \
-		< config/manifests/bases/kuadrant-operator.clusterserviceversion.template.yaml \
-		> config/manifests/bases/kuadrant-operator.clusterserviceversion.yaml
+	# Update CSV
+	V="kuadrant-operator.v$(VERSION)" $(YQ) eval '.metadata.name = strenv(V)' -i config/manifests/bases/kuadrant-operator.clusterserviceversion.yaml
+	V="$(VERSION)" $(YQ) eval '.spec.version = strenv(V)' -i config/manifests/bases/kuadrant-operator.clusterserviceversion.yaml
+	V="$(IMG)" $(YQ) eval '.metadata.annotations.containerImage = strenv(V)' -i config/manifests/bases/kuadrant-operator.clusterserviceversion.yaml
+	# Generate bundle
 	$(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS)
+	# Update operator dependencies
+	# TODO(eguzki): run only if not default one. Avoid bundle parsing if version is known in advance
+	V=`$(PROJECT_PATH)/utils/parse-bundle-version.sh $(OPM) $(YQ) $(LIMITADOR_OPERATOR_BUNDLE_IMG)` \
+	    $(YQ) eval '(.dependencies[] | select(.value.packageName == "limitador-operator").value.version) = strenv(V)' -i bundle/metadata/dependencies.yaml
+	V=`$(PROJECT_PATH)/utils/parse-bundle-version.sh $(OPM) $(YQ) $(AUTHORINO_OPERATOR_BUNDLE_IMG)` \
+	    $(YQ) eval '(.dependencies[] | select(.value.packageName == "authorino-operator").value.version) = strenv(V)' -i bundle/metadata/dependencies.yaml
+	# Validate bundle manifests
 	$(OPERATOR_SDK) bundle validate ./bundle
 
 .PHONY: bundle-build
@@ -321,51 +364,6 @@ bundle-build: ## Build the bundle image.
 bundle-push: ## Push the bundle image.
 	$(MAKE) docker-push IMG=$(BUNDLE_IMG)
 
-.PHONY: opm
-OPM = ./bin/opm
-opm: ## Download opm locally if necessary.
-ifeq (,$(wildcard $(OPM)))
-ifeq (,$(shell which opm 2>/dev/null))
-	@{ \
-	set -e ;\
-	mkdir -p $(dir $(OPM)) ;\
-	OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
-	curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$${OS}-$${ARCH}-opm ;\
-	chmod +x $(OPM) ;\
-	}
-else
-OPM = $(shell which opm)
-endif
-endif
-
-# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0).
-# These images MUST exist in a registry and be pull-able.
-BUNDLE_IMGS ?= $(BUNDLE_IMG),$(LIMITADOR_OPERATOR_BUNDLE_IMG),$(AUTHORINO_OPERATOR_BUNDLE_IMG)
-
-# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0).
-CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:$(IMAGE_TAG)
-
-# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image.
-ifneq ($(origin CATALOG_BASE_IMG), undefined)
-FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG)
-endif
-
-# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'.
-# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see:
-# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator
-.PHONY: catalog-build
-catalog-build: opm ## Build a catalog image.
-	$(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
-
-.PHONY: catalog-generate
-catalog-generate: opm ## Generate a catalog/index Dockerfile.
-	$(OPM) index add --generate --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT)
-
-# Push the catalog image.
-.PHONY: catalog-push
-catalog-push: ## Push a catalog image.
-	$(MAKE) docker-push IMG=$(CATALOG_IMG)
-
 ##@ Code Style
 
 GOLANGCI-LINT = $(PROJECT_PATH)/bin/golangci-lint

catalog/authorino-operator-channel-entry.yaml

@@ -0,0 +1,6 @@
+---
+schema: olm.channel
+package: authorino-operator
+name: preview
+entries:
+  - name: authorino-operator.v0.0.0

catalog/kuadrant-operator-channel-entry.yaml

@@ -0,0 +1,6 @@
+---
+schema: olm.channel
+package: kuadrant-operator
+name: preview
+entries:
+  - name: kuadrant-operator.v0.0.0

catalog/limitador-operator-channel-entry.yaml

@@ -0,0 +1,6 @@
+---
+schema: olm.channel
+package: limitador-operator
+name: preview
+entries:
+  - name: limitador-operator.v0.0.0

config/deploy/olm/subscription.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: operators.coreos.com/v1alpha1
 kind: Subscription
 metadata:
@@ -6,4 +7,4 @@ spec:
   source: kuadrant-operator-catalog
   sourceNamespace: kuadrant-system
   name: kuadrant-operator
-  channel: "alpha"
+  channel: "preview"