refactor docker and change buildkit-syft-scanner reference to ghcr.io (…

…#2577)

Signed-off-by: kpango <kpango@vdaas.org>

.gitfiles

@@ -85,6 +85,7 @@
 .github/workflows/dockers-binfmt-image.yaml
 .github/workflows/dockers-buildbase-image.yml
 .github/workflows/dockers-buildkit-image.yaml
+.github/workflows/dockers-buildkit-syft-scanner-image.yaml
 .github/workflows/dockers-ci-container-image.yml
 .github/workflows/dockers-dev-container-image.yml
 .github/workflows/dockers-discoverer-k8s-image.yml
@@ -292,17 +293,11 @@ assets/test/templates/common/call.tmpl
 assets/test/templates/common/fill.tmpl
 assets/test/templates/common/function.tmpl
 assets/test/templates/common/header.tmpl
-assets/test/templates/common/inline.tmpl
-assets/test/templates/common/inputs.tmpl
-assets/test/templates/common/message.tmpl
 assets/test/templates/common/results.tmpl
 assets/test/templates/option/call.tmpl
 assets/test/templates/option/fill.tmpl
 assets/test/templates/option/function.tmpl
 assets/test/templates/option/header.tmpl
-assets/test/templates/option/inline.tmpl
-assets/test/templates/option/inputs.tmpl
-assets/test/templates/option/message.tmpl
 assets/test/templates/option/results.tmpl
 buf.gen.yaml
 buf.work.yaml
@@ -516,7 +511,6 @@ cmd/tools/cli/loadtest/main_test.go
 cmd/tools/cli/loadtest/sample.yaml
 cmd/tools/cli/vdctl/main.go
 cmd/tools/cli/vdctl/main_test.go
-design/.gitkeep
 design/Vald Architecture Assets.drawio
 design/Vald Architecture Dataflow.drawio
 design/Vald Architecture Overview.drawio
@@ -531,6 +525,7 @@ dockers/agent/sidecar/README.md
 dockers/binfmt/Dockerfile
 dockers/buildbase/Dockerfile
 dockers/buildkit/Dockerfile
+dockers/buildkit/syft/scanner/Dockerfile
 dockers/ci/base/Dockerfile
 dockers/ci/base/README.md
 dockers/dev/Dockerfile
@@ -1403,85 +1398,41 @@ internal/worker/worker.go
 internal/worker/worker_option.go
 internal/worker/worker_option_test.go
 internal/worker/worker_test.go
-k8s/agent/clusterrole.yaml
-k8s/agent/clusterrolebinding.yaml
-k8s/agent/daemonset.yaml
-k8s/agent/deployment.yaml
-k8s/agent/faiss/configmap.yaml
-k8s/agent/hpa.yaml
-k8s/agent/networkpolicy.yaml
 k8s/agent/ngt/configmap.yaml
 k8s/agent/pdb.yaml
 k8s/agent/priorityclass.yaml
-k8s/agent/serviceaccount.yaml
-k8s/agent/sidecar/configmap.yaml
-k8s/agent/sidecar/svc.yaml
 k8s/agent/statefulset.yaml
 k8s/agent/svc.yaml
 k8s/debug/kind/config.yaml
 k8s/discoverer/clusterrole.yaml
 k8s/discoverer/clusterrolebinding.yaml
 k8s/discoverer/configmap.yaml
-k8s/discoverer/daemonset.yaml
 k8s/discoverer/deployment.yaml
-k8s/discoverer/hpa.yaml
-k8s/discoverer/networkpolicy.yaml
 k8s/discoverer/pdb.yaml
 k8s/discoverer/priorityclass.yaml
 k8s/discoverer/serviceaccount.yaml
 k8s/discoverer/svc.yaml
 k8s/external/minio/deployment.yaml
 k8s/external/minio/mb-job.yaml
 k8s/external/minio/svc.yaml
-k8s/gateway/gateway/filter/configmap.yaml
-k8s/gateway/gateway/filter/daemonset.yaml
-k8s/gateway/gateway/filter/deployment.yaml
-k8s/gateway/gateway/filter/hpa.yaml
-k8s/gateway/gateway/filter/networkpolicy.yaml
-k8s/gateway/gateway/filter/pdb.yaml
-k8s/gateway/gateway/filter/priorityclass.yaml
-k8s/gateway/gateway/filter/svc.yaml
 k8s/gateway/gateway/ing.yaml
 k8s/gateway/gateway/lb/configmap.yaml
-k8s/gateway/gateway/lb/daemonset.yaml
 k8s/gateway/gateway/lb/deployment.yaml
 k8s/gateway/gateway/lb/hpa.yaml
-k8s/gateway/gateway/lb/networkpolicy.yaml
 k8s/gateway/gateway/lb/pdb.yaml
 k8s/gateway/gateway/lb/priorityclass.yaml
 k8s/gateway/gateway/lb/svc.yaml
-k8s/gateway/gateway/mirror/clusterrole.yaml
-k8s/gateway/gateway/mirror/clusterrolebinding.yaml
-k8s/gateway/gateway/mirror/configmap.yaml
-k8s/gateway/gateway/mirror/daemonset.yaml
-k8s/gateway/gateway/mirror/deployment.yaml
-k8s/gateway/gateway/mirror/hpa.yaml
-k8s/gateway/gateway/mirror/networkpolicy.yaml
-k8s/gateway/gateway/mirror/pdb.yaml
-k8s/gateway/gateway/mirror/priorityclass.yaml
-k8s/gateway/gateway/mirror/serviceaccount.yaml
-k8s/gateway/gateway/mirror/svc.yaml
 k8s/index/job/correction/configmap.yaml
 k8s/index/job/correction/cronjob.yaml
-k8s/index/job/correction/networkpolicy.yaml
 k8s/index/job/creation/configmap.yaml
 k8s/index/job/creation/cronjob.yaml
-k8s/index/job/creation/networkpolicy.yaml
-k8s/index/job/readreplica/rotate/clusterrole.yaml
-k8s/index/job/readreplica/rotate/clusterrolebinding.yaml
-k8s/index/job/readreplica/rotate/configmap.yaml
-k8s/index/job/readreplica/rotate/networkpolicy.yaml
-k8s/index/job/readreplica/rotate/serviceaccount.yaml
 k8s/index/job/save/configmap.yaml
 k8s/index/job/save/cronjob.yaml
-k8s/index/job/save/networkpolicy.yaml
 k8s/index/operator/configmap.yaml
 k8s/index/operator/deployment.yaml
 k8s/index/operator/priorityclass.yaml
 k8s/manager/index/configmap.yaml
-k8s/manager/index/daemonset.yaml
 k8s/manager/index/deployment.yaml
-k8s/manager/index/networkpolicy.yaml
 k8s/manager/index/pdb.yaml
 k8s/manager/index/priorityclass.yaml
 k8s/manager/index/svc.yaml
@@ -1540,12 +1491,6 @@ k8s/operator/helm/operator.yaml
 k8s/operator/helm/serviceaccount.yaml
 k8s/operator/helm/svc.yaml
 k8s/readreplica/configmap.yaml
-k8s/readreplica/deployment.yaml
-k8s/readreplica/hpa.yaml
-k8s/readreplica/networkpolicy.yaml
-k8s/readreplica/pvc.yaml
-k8s/readreplica/snapshot.yaml
-k8s/readreplica/svc.yaml
 k8s/tools/benchmark/job/clusterrole.yaml
 k8s/tools/benchmark/job/clusterrolebinding.yaml
 k8s/tools/benchmark/job/serviceaccount.yaml
@@ -1960,29 +1905,21 @@ rust/libs/ngt-rs/src/lib.rs
 rust/libs/ngt/Cargo.toml
 rust/libs/ngt/src/lib.rs
 rust/libs/proto/Cargo.toml
-rust/libs/proto/src/core.v1.rs
 rust/libs/proto/src/core.v1.tonic.rs
-rust/libs/proto/src/discoverer.v1.rs
 rust/libs/proto/src/discoverer.v1.tonic.rs
-rust/libs/proto/src/filter.egress.v1.rs
 rust/libs/proto/src/filter.egress.v1.tonic.rs
-rust/libs/proto/src/filter.ingress.v1.rs
 rust/libs/proto/src/filter.ingress.v1.tonic.rs
 rust/libs/proto/src/lib.rs
-rust/libs/proto/src/mirror.v1.rs
 rust/libs/proto/src/mirror.v1.tonic.rs
 rust/libs/proto/src/payload.v1.rs
 rust/libs/proto/src/rpc.v1.rs
-rust/libs/proto/src/sidecar.v1.rs
 rust/libs/proto/src/sidecar.v1.tonic.rs
-rust/libs/proto/src/vald.v1.rs
 rust/libs/proto/src/vald.v1.tonic.rs
 rust/rust-toolchain
 rust/rust-toolchain.toml
 tests/chaos/chart/.helmignore
 tests/chaos/chart/Chart.yaml
 tests/chaos/chart/README.md
-tests/chaos/chart/templates/NOTES.txt
 tests/chaos/chart/templates/_helpers.tpl
 tests/chaos/chart/templates/network/bandwidth.yaml
 tests/chaos/chart/templates/network/partition.yaml

.github/workflows/_docker-image.yaml

@@ -48,8 +48,9 @@ jobs:
           (github.event_name == 'push' &&
               github.ref == 'refs/heads/main') ||
           (github.event_name == 'push' &&
-              startsWith( github.ref, 'refs/heads/release/v')) ||
-          startsWith( github.ref, 'refs/tags/')
+              startsWith(github.ref, 'refs/heads/release/v')) ||
+          startsWith(github.ref, 'refs/tags/') ||
+          (github.event_name == 'schedule')
       }}
     steps:
       - name: Get ref
@@ -80,7 +81,7 @@ jobs:
           driver-opts: |
             image=ghcr.io/vdaas/vald/vald-buildkit:nightly
             network=host
-          buildkitd-flags: "--debug --oci-worker-gc=false"
+          buildkitd-flags: "--debug --oci-worker-gc=false --oci-worker-snapshotter=stargz"
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:

Makefile

@@ -24,15 +24,15 @@ TAG                            ?= latest
 CRORG                          ?= $(ORG)
 GHCRORG                         = ghcr.io/$(REPO)
 AGENT_IMAGE                     = $(NAME)-agent
-AGENT_NGT_IMAGE                 = $(NAME)-agent-ngt
-AGENT_FAISS_IMAGE               = $(NAME)-agent-faiss
-AGENT_SIDECAR_IMAGE             = $(NAME)-agent-sidecar
+AGENT_NGT_IMAGE                 = $(AGENT_IMAGE)-ngt
+AGENT_FAISS_IMAGE               = $(AGENT_IMAGE)-faiss
+AGENT_SIDECAR_IMAGE             = $(AGENT_IMAGE)-sidecar
 BENCHMARK_JOB_IMAGE             = $(NAME)-benchmark-job
 BENCHMARK_OPERATOR_IMAGE        = $(NAME)-benchmark-operator
 BINFMT_IMAGE                    = $(NAME)-binfmt
 BUILDBASE_IMAGE                 = $(NAME)-buildbase
 BUILDKIT_IMAGE                  = $(NAME)-buildkit
-BUILDKIT_SYFT_SCANNER_IMAGE     = $(NAME)-buildkit-syft-scanner
+BUILDKIT_SYFT_SCANNER_IMAGE     = $(BUILDKIT_IMAGE)-syft-scanner
 CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 DEV_CONTAINER_IMAGE             = $(NAME)-dev-container
 DISCOVERER_IMAGE                = $(NAME)-discoverer-k8s
@@ -49,6 +49,8 @@ MIRROR_GATEWAY_IMAGE            = $(NAME)-mirror-gateway
 READREPLICA_ROTATE_IMAGE        = $(NAME)-readreplica-rotate
 MAINTAINER                      = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"
 
+DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE = $(GHCRORG)/$(BUILDKIT_SYFT_SCANNER_IMAGE):nightly
+
 VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)
 
 NGT_REPO = github.com/yahoojapan/NGT

Makefile.d/docker.mk

@@ -68,7 +68,7 @@ ifeq ($(REMOTE),true)
 		--build-arg GO_VERSION=$(GO_VERSION) \
 		--build-arg RUST_VERSION=$(RUST_VERSION) \
 		--build-arg MAINTAINER=$(MAINTAINER) \
-		--attest type=sbom,generator=docker/buildkit-syft-scanner:edge \
+		--attest type=sbom,generator=$(DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE) \
 		--provenance=mode=max \
 		-t $(CRORG)/$(IMAGE):$(TAG) \
 		-t $(GHCRORG)/$(IMAGE):$(TAG) \
@@ -229,6 +229,7 @@ docker/name/buildkit-syft-scanner:
 docker/build/buildkit-syft-scanner:
 	@make DOCKERFILE="$(ROOTDIR)/dockers/buildkit/syft/scanner/Dockerfile" \
 		IMAGE=$(BUILDKIT_SYFT_SCANNER_IMAGE) \
+		DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE="docker/buildkit-syft-scanner:edge" \
 		docker/build/image
 
 .PHONY: docker/name/ci-container

dockers/agent/core/agent/Dockerfile

@@ -15,17 +15,13 @@
 # limitations under the License.
 #
 
-# DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go
-
+# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
 ARG UPX_OPTIONS=-9
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
 # skipcq: DOK-DL3002
 USER root:root
-
 ARG TARGETARCH
 ARG TARGETOS
 ARG GO_VERSION
@@ -43,10 +39,9 @@ ENV REPO=vald
 ENV RUST_HOME=/usr/loacl/lib/rust
 ENV TZ=Etc/UTC
 ENV USER=root
-ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV CARGO_HOME=${RUST_HOME}/cargo
+ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}
-
 WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO}
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 #skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
@@ -92,12 +87,8 @@ RUN --mount=type=bind,target=.,rw \
     && rm -rf rust/target
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM gcr.io/distroless/cc-debian12:nonroot
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
-ENV APP_NAME=agent
-
-COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
+COPY --from=builder /usr/bin/agent /usr/bin/agent
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/agent"]
+ENTRYPOINT ["/usr/bin/agent"]

dockers/agent/core/faiss/Dockerfile

@@ -15,17 +15,13 @@
 # limitations under the License.
 #
 
-# DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go
-
+# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
 ARG UPX_OPTIONS=-9
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
 # skipcq: DOK-DL3002
 USER root:root
-
 ARG TARGETARCH
 ARG TARGETOS
 ARG GO_VERSION
@@ -46,7 +42,6 @@ ENV REPO=vald
 ENV TZ=Etc/UTC
 ENV USER=root
 ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH}
-
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 #skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
@@ -93,13 +88,9 @@ RUN --mount=type=bind,target=.,rw \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM gcr.io/distroless/static:nonroot
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
-ENV APP_NAME=faiss
-
-COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
+COPY --from=builder /usr/bin/faiss /usr/bin/faiss
 COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/faiss"]
+ENTRYPOINT ["/usr/bin/faiss"]

dockers/agent/core/ngt/Dockerfile

@@ -15,17 +15,13 @@
 # limitations under the License.
 #
 
-# DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go
-
+# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
 ARG UPX_OPTIONS=-9
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
 # skipcq: DOK-DL3002
 USER root:root
-
 ARG TARGETARCH
 ARG TARGETOS
 ARG GO_VERSION
@@ -46,7 +42,6 @@ ENV REPO=vald
 ENV TZ=Etc/UTC
 ENV USER=root
 ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH}
-
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 #skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
@@ -92,13 +87,9 @@ RUN --mount=type=bind,target=.,rw \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM gcr.io/distroless/static:nonroot
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
-ENV APP_NAME=ngt
-
-COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
+COPY --from=builder /usr/bin/ngt /usr/bin/ngt
 COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/ngt"]
+ENTRYPOINT ["/usr/bin/ngt"]