diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 19f8a01c6..4f954b781 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -67,8 +67,10 @@ jobs:
- name: container
run: |
- make ko-local 2>&1 | tee output.txt
- docker run --rm $(tail -1 output.txt) version
+ make ko-local
+ docker run --rm $(cat rekorImagerefs) version
+ docker run --rm $(cat cliImagerefs) version
+ docker run --rm $(cat redisImagerefs) --version
e2e:
runs-on: ubuntu-20.04
diff --git a/.gitignore b/.gitignore
index 55fce30e7..dc1389003 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,9 @@ rekorServerImagerefs
rekorCliImagerefs
trillianServerImagerefs
trillianSignerImagerefs
+rekorImagerefs
+cliImagerefs
+redisImagerefs
cosign.*
signature
rekor.pub
diff --git a/.ko.yaml b/.ko.yaml
index ecf440910..18c35c779 100644
--- a/.ko.yaml
+++ b/.ko.yaml
@@ -46,3 +46,18 @@ builds:
ldflags:
- -extldflags "-static"
- "{{ .Env.LDFLAGS }}"
+
+- id: backfill-redis
+ dir: .
+ main: ./cmd/backfill-redis
+ env:
+ - CGO_ENABLED=0
+ flags:
+ - -trimpath
+ - --tags
+ - "{{ .Env.GIT_HASH }}"
+ - --tags
+ - "{{ .Env.GIT_VERSION }}"
+ ldflags:
+ - -extldflags "-static"
+ - "{{ .Env.LDFLAGS }}"
diff --git a/Makefile b/Makefile
index ca25f716d..4a4e9ced9 100644
--- a/Makefile
+++ b/Makefile
@@ -81,6 +81,9 @@ rekor-cli: $(SRCS)
rekor-server: $(SRCS)
CGO_ENABLED=0 go build -trimpath -ldflags "$(SERVER_LDFLAGS)" -o rekor-server ./cmd/rekor-server
+backfill-redis: $(SRCS)
+ CGO_ENABLED=0 go build -trimpath -ldflags "$(SERVER_LDFLAGS)" -o rekor-server ./cmd/backfill-redis
+
test:
go test ./...
@@ -122,6 +125,12 @@ ko:
--platform=all --tags $(GIT_VERSION) --tags $(GIT_HASH) \
--image-refs rekorCliImagerefs github.com/sigstore/rekor/cmd/rekor-cli
+ # backfill-redis
+ LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
+ ko publish --base-import-paths \
+ --platform=all --tags $(GIT_VERSION) --tags $(GIT_HASH) \
+ --image-refs bRedisImagerefs github.com/sigstore/rekor/cmd/backfill-redis
+
deploy:
LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) ko apply -f config/
@@ -138,14 +147,19 @@ sign-keyless-ci: ko
ko-local:
LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
ko publish --base-import-paths \
- --tags $(GIT_VERSION) --tags $(GIT_HASH) --local \
+ --tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs rekorImagerefs \
github.com/sigstore/rekor/cmd/rekor-server
LDFLAGS="$(CLI_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
ko publish --base-import-paths \
- --tags $(GIT_VERSION) --tags $(GIT_HASH) --local \
+ --tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs cliImagerefs \
github.com/sigstore/rekor/cmd/rekor-cli
+ LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
+ ko publish --base-import-paths \
+ --tags $(GIT_VERSION) --tags $(GIT_HASH) --local --image-refs redisImagerefs \
+ github.com/sigstore/rekor/cmd/backfill-redis
+
# This builds the trillian containers we rely on using ko for cross platform support
.PHONY: ko-trillian
ko-trillian:
diff --git a/cmd/backfill-redis/main.go b/cmd/backfill-redis/main.go
new file mode 100644
index 000000000..58f3c1797
--- /dev/null
+++ b/cmd/backfill-redis/main.go
@@ -0,0 +1,181 @@
+// Copyright 2022 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+ backfill-redis is a script to populate the Redis index with entries
+ from Rekor. This is sometimes necessary because Redis caching is best
+ effort. If Redis returns an error, Rekor will not, and so sometimes
+ we need to backfill missing entries into Redis for the search API.
+
+ To run:
+ go run cmd/backfill-redis/main.go --rekor-address
\
+ --hostname --port
+ --start --end
+*/
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "flag"
+ "fmt"
+ "log"
+ "os"
+
+ "github.com/go-openapi/runtime"
+ radix "github.com/mediocregopher/radix/v4"
+ "sigs.k8s.io/release-utils/version"
+
+ "github.com/sigstore/rekor/pkg/client"
+ "github.com/sigstore/rekor/pkg/generated/client/entries"
+ "github.com/sigstore/rekor/pkg/generated/models"
+ "github.com/sigstore/rekor/pkg/types"
+
+ // these imports are to call the packages' init methods
+ _ "github.com/sigstore/rekor/pkg/types/alpine/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/cose/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/hashedrekord/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/helm/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.1"
+
+ // remove 0.0.2 intoto type due to bugs
+ // _ "github.com/sigstore/rekor/pkg/types/intoto/v0.0.2"
+ _ "github.com/sigstore/rekor/pkg/types/jar/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/rekord/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/rfc3161/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/rpm/v0.0.1"
+ _ "github.com/sigstore/rekor/pkg/types/tuf/v0.0.1"
+)
+
+var (
+ redisHostname = flag.String("hostname", "", "Hostname for Redis application")
+ redisPort = flag.String("port", "", "Port to Redis application")
+ startIndex = flag.Int("start", -1, "First index to backfill")
+ endIndex = flag.Int("end", -1, "Last index to backfill")
+ rekorAddress = flag.String("rekor-address", "", "Address for Rekor, e.g. https://rekor.sigstore.dev")
+ versionFlag = flag.Bool("version", false, "Print the current version of Backfill Redis")
+)
+
+func main() {
+ flag.Parse()
+
+ versionInfo := version.GetVersionInfo()
+ if *versionFlag {
+ fmt.Println(versionInfo.String())
+ os.Exit(0)
+ }
+
+ if *redisHostname == "" {
+ log.Fatal("address must be set")
+ }
+ if *redisPort == "" {
+ log.Fatal("port must be set")
+ }
+ if *startIndex == -1 {
+ log.Fatal("start must be set to >=0")
+ }
+ if *endIndex == -1 {
+ log.Fatal("end must be set to >=0")
+ }
+ if *rekorAddress == "" {
+ log.Fatal("rekor-address must be set")
+ }
+
+ log.Printf("running backfill redis Version: %s GitCommit: %s BuildDate: %s", versionInfo.GitVersion, versionInfo.GitCommit, versionInfo.BuildDate)
+
+ cfg := radix.PoolConfig{}
+ redisClient, err := cfg.New(context.Background(), "tcp", fmt.Sprintf("%s:%s", *redisHostname, *redisPort))
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ rekorClient, err := client.GetRekorClient(*rekorAddress)
+ if err != nil {
+ log.Fatalf("creating rekor client: %v", err)
+ }
+
+ for i := *startIndex; i <= *endIndex; i++ {
+ params := entries.NewGetLogEntryByIndexParamsWithContext(context.Background())
+ params.SetLogIndex(int64(i))
+ resp, err := rekorClient.Entries.GetLogEntryByIndex(params)
+ if err != nil {
+ log.Fatalf("retrieving log uuid by index: %v", err)
+ }
+ var insertErrs []error
+ for uuid, entry := range resp.Payload {
+ // uuid is the global UUID - tree ID and entry UUID
+ e, _, _, err := unmarshalEntryImpl(entry.Body.(string))
+ if err != nil {
+ insertErrs = append(insertErrs, fmt.Errorf("error unmarshalling entry for %s: %v", uuid, err))
+ continue
+ }
+ keys, err := e.IndexKeys()
+ if err != nil {
+ insertErrs = append(insertErrs, fmt.Errorf("error building index keys for %s: %v", uuid, err))
+ continue
+ }
+ for _, key := range keys {
+ // remove the key-value pair from the index in case it already exists
+ if err := removeFromIndex(context.Background(), redisClient, key, uuid); err != nil {
+ insertErrs = append(insertErrs, fmt.Errorf("error removing UUID %s with key %s: %v", uuid, key, err))
+ }
+ if err := addToIndex(context.Background(), redisClient, key, uuid); err != nil {
+ insertErrs = append(insertErrs, fmt.Errorf("error inserting UUID %s with key %s: %v", uuid, key, err))
+ }
+ fmt.Printf("Uploaded Redis entry %s, index %d, key %s\n", uuid, i, key)
+ }
+ }
+ if len(insertErrs) != 0 {
+ fmt.Printf("Errors with log index %d:\n", i)
+ for _, e := range insertErrs {
+ fmt.Println(e)
+ }
+ } else {
+ fmt.Printf("Completed log index %d\n", i)
+ }
+ }
+}
+
+// unmarshalEntryImpl decodes the base64-encoded entry to a specific entry type (types.EntryImpl).
+// Taken from Cosign
+func unmarshalEntryImpl(e string) (types.EntryImpl, string, string, error) {
+ b, err := base64.StdEncoding.DecodeString(e)
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ pe, err := models.UnmarshalProposedEntry(bytes.NewReader(b), runtime.JSONConsumer())
+ if err != nil {
+ return nil, "", "", err
+ }
+
+ entry, err := types.UnmarshalEntry(pe)
+ if err != nil {
+ return nil, "", "", err
+ }
+ return entry, pe.Kind(), entry.APIVersion(), nil
+}
+
+// removeFromIndex removes all occurrences of a value from a given key. This guards against
+// multiple invocations of backfilling creating duplicates.
+func removeFromIndex(ctx context.Context, redisClient radix.Client, key, value string) error {
+ return redisClient.Do(ctx, radix.Cmd(nil, "LREM", key, "0", value))
+}
+
+// addToIndex pushes a value onto a key of type list.
+func addToIndex(ctx context.Context, redisClient radix.Client, key, value string) error {
+ return redisClient.Do(ctx, radix.Cmd(nil, "LPUSH", key, value))
+}
diff --git a/release/ko-sign-release-images.sh b/release/ko-sign-release-images.sh
index ca6b6f177..1688d0e77 100755
--- a/release/ko-sign-release-images.sh
+++ b/release/ko-sign-release-images.sh
@@ -36,6 +36,11 @@ if [[ ! -f rekorCliImagerefs ]]; then
exit 1
fi
+if [[ ! -f bRedisImagerefs ]]; then
+ echo "bRedisImagerefs not found"
+ exit 1
+fi
+
if [[ ! -f trillianServerImagerefs ]]; then
echo "trillianServerImagerefs not found"
exit 1
@@ -49,11 +54,13 @@ fi
echo "Signing images with GCP KMS Key..."
cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorServerImagerefs)
cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorCliImagerefs)
+cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat bRedisImagerefs)
cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianServerImagerefs)
cosign sign --force --key "gcpkms://projects/$PROJECT_ID/locations/$KEY_LOCATION/keyRings/$KEY_RING/cryptoKeys/$KEY_NAME/versions/$KEY_VERSION" -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianSignerImagerefs)
echo "Signing images with Keyless..."
cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorServerImagerefs)
cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat rekorCliImagerefs)
+cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat bRedisImagerefs)
cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianServerImagerefs)
cosign sign --force -a GIT_HASH="$GIT_HASH" -a GIT_VERSION="$GIT_VERSION" $(cat trillianSignerImagerefs)
diff --git a/release/release.mk b/release/release.mk
index 19dc18209..19a1c6d63 100644
--- a/release/release.mk
+++ b/release/release.mk
@@ -5,12 +5,12 @@
# used when releasing together with GCP CloudBuild
.PHONY: release
release:
- CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --rm-dist --timeout 60m
+ CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --rm-dist --timeout 120m
# used when need to validate the goreleaser
.PHONY: snapshot
snapshot:
- CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --skip-sign --skip-publish --snapshot --rm-dist
+ CLI_LDFLAGS="$(CLI_LDFLAGS)" SERVER_LDFLAGS="$(SERVER_LDFLAGS)" goreleaser release --skip-sign --skip-publish --snapshot --rm-dist --timeout 120m
###########################
# sign section
@@ -33,6 +33,10 @@ copy-rekor-server-signed-release-to-ghcr:
copy-rekor-cli-signed-release-to-ghcr:
cosign copy $(KO_PREFIX)/rekor-cli:$(GIT_VERSION) $(GHCR_PREFIX)/rekor-cli:$(GIT_VERSION)
+.PHONY: copy-backfill-redis-signed-release-to-ghcr
+copy-backfill-redis-signed-release-to-ghcr:
+ cosign copy $(KO_PREFIX)/backfill-redis:$(GIT_VERSION) $(GHCR_PREFIX)/backfill-redis:$(GIT_VERSION)
+
.PHONY: copy-trillian-log-server-signed-release-to-ghcr
copy-trillian-log-server-signed-release-to-ghcr:
cosign copy $(KO_PREFIX)/trillian_log_server:$(GIT_VERSION) $(GHCR_PREFIX)/trillian_log_server:$(GIT_VERSION)
@@ -42,7 +46,7 @@ copy-trillian-log-signer-signed-release-to-ghcr:
cosign copy $(KO_PREFIX)/trillian_log_signer:$(GIT_VERSION) $(GHCR_PREFIX)/trillian_log_signer:$(GIT_VERSION)
.PHONY: copy-signed-release-to-ghcr
-copy-signed-release-to-ghcr: copy-rekor-server-signed-release-to-ghcr copy-rekor-cli-signed-release-to-ghcr copy-trillian-log-signer-signed-release-to-ghcr copy-trillian-log-server-signed-release-to-ghcr
+copy-signed-release-to-ghcr: copy-rekor-server-signed-release-to-ghcr copy-rekor-cli-signed-release-to-ghcr copy-backfill-redis-signed-release-to-ghcr copy-trillian-log-signer-signed-release-to-ghcr copy-trillian-log-server-signed-release-to-ghcr
## --------------------------------------
## Dist / maybe we can deprecate