Add ignore images flag on generate missing images list (#463)

* implement viper

* add validator

* remove viper and add tests

* remove logs

* fix flags

* fix md

* fix install docs

* fix typo on dry run

* rename confbytes to b

* add docs to functions

* add ignore images and check images flag

* fix command name

cmd/release/cmd/generate.go

@@ -21,6 +21,9 @@ var (
 
 	concurrencyLimit                    int
 	imagesListURL                       string
+	ignoreImages                        []string
+	checkImages                         []string
+	registry                            string
 	rancherMissingImagesJSONOutput      bool
 	rke2PrevMilestone                   string
 	rke2Milestone                       string
@@ -115,19 +118,13 @@ var rancherGenerateArtifactsIndexSubCmd = &cobra.Command{
 }
 
 var rancherGenerateMissingImagesListSubCmd = &cobra.Command{
-	Use:   "missing-images-list [version]",
+	Use:   "missing-images-list",
 	Short: "Generate a missing images list",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		if len(args) < 1 {
-			return errors.New("expected at least one argument: [version]")
-		}
-		checkImages := make([]string, 0)
-		version := args[0]
-		rancherRelease, found := rootConfig.Rancher.Versions[version]
-		if found {
-			checkImages = rancherRelease.CheckImages
+		if len(checkImages) == 0 && imagesListURL == "" {
+			return errors.New("either --images-list-url or --check-images must be provided")
 		}
-		missingImages, err := rancher.GenerateMissingImagesList(version, imagesListURL, concurrencyLimit, checkImages)
+		missingImages, err := rancher.GenerateMissingImagesList(imagesListURL, registry, concurrencyLimit, checkImages, ignoreImages)
 		if err != nil {
 			return err
 		}
@@ -204,10 +201,9 @@ func init() {
 	rancherGenerateMissingImagesListSubCmd.Flags().IntVarP(&concurrencyLimit, "concurrency-limit", "l", 3, "Concurrency Limit")
 	rancherGenerateMissingImagesListSubCmd.Flags().BoolVarP(&rancherMissingImagesJSONOutput, "json", "j", false, "JSON Output")
 	rancherGenerateMissingImagesListSubCmd.Flags().StringVarP(&imagesListURL, "images-list-url", "i", "", "URL of the artifact containing all images for a given version 'rancher-images.txt' (required)")
-	if err := rancherGenerateMissingImagesListSubCmd.MarkFlagRequired("images-list-url"); err != nil {
-		fmt.Println(err.Error())
-		os.Exit(1)
-	}
+	rancherGenerateMissingImagesListSubCmd.Flags().StringSliceVarP(&ignoreImages, "ignore-images", "g", make([]string, 0), "Images to ignore when checking for missing images without the version. e.g: rancher/rancher")
+	rancherGenerateMissingImagesListSubCmd.Flags().StringSliceVarP(&checkImages, "check-images", "k", make([]string, 0), "Images to check for when checking for missing images with the version. e.g: rancher/rancher-agent:v2.9.0")
+	rancherGenerateMissingImagesListSubCmd.Flags().StringVarP(&registry, "registry", "r", "registry.rancher.com", "Registry where the images should be located at")
 
 	// rancher generate docker-images-digests
 	rancherGenerateDockerImagesDigestsSubCmd.Flags().StringVarP(&rancherImagesDigestsOutputFile, "output-file", "o", "", "Output file with images digests")

cmd/release/cmd/root.go

@@ -43,7 +43,7 @@ func SetVersion(version string) {
 
 func init() {
 	rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "D", false, "Debug")
-	rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "R", false, "Drun Run")
+	rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "R", false, "Dry Run")
 	rootCmd.PersistentFlags().BoolVarP(&ignoreValidate, "ignore-validate", "I", false, "Ignore the validate config step")
 	rootCmd.PersistentFlags().StringVarP(&configFile, "config-file", "c", "$HOME/.ecm-distro-tools/config.json", "Path for the config.json file")
 	rootCmd.PersistentFlags().StringVarP(&stringConfig, "config", "C", "", "JSON config string")

cmd/release/config/config.go

@@ -31,15 +31,14 @@ type K3sRelease struct {
 
 // RancherRelease
 type RancherRelease struct {
-	ReleaseBranch        string   `json:"release_branch" validate:"required"`
-	RancherRepoOwner     string   `json:"rancher_repo_owner" validate:"required"`
-	IssueNumber          string   `json:"issue_number" validate:"number"`
-	CheckImages          []string `json:"check_images" validate:"required"`
-	BaseRegistry         string   `json:"base_registry" validate:"required,hostname"`
-	Registry             string   `json:"registry" validate:"required,hostname"`
-	PrimeArtifactsBucket string   `json:"prime_artifacts_bucket" validate:"required"`
-	DryRun               bool     `json:"dry_run"`
-	SkipStatusCheck      bool     `json:"skip_status_check"`
+	ReleaseBranch        string `json:"release_branch" validate:"required"`
+	RancherRepoOwner     string `json:"rancher_repo_owner" validate:"required"`
+	IssueNumber          string `json:"issue_number" validate:"number"`
+	BaseRegistry         string `json:"base_registry" validate:"required,hostname"`
+	Registry             string `json:"registry" validate:"required,hostname"`
+	PrimeArtifactsBucket string `json:"prime_artifacts_bucket" validate:"required"`
+	DryRun               bool   `json:"dry_run"`
+	SkipStatusCheck      bool   `json:"skip_status_check"`
 }
 
 // RKE2
@@ -165,7 +164,6 @@ func ExampleConfig() (string, error) {
 					DryRun:               false,
 					SkipStatusCheck:      false,
 					RancherRepoOwner:     "rancher",
-					CheckImages:          []string{},
 					IssueNumber:          "1234",
 					BaseRegistry:         "stgregistry.suse.com",
 					Registry:             "registry.rancher.com",

release/rancher/rancher.go

@@ -386,46 +386,44 @@ func formatContentLine(line string) string {
 	return strings.TrimSpace(line)
 }
 
-func GenerateMissingImagesList(version, imagesListURL string, concurrencyLimit int, images []string) ([]string, error) {
-	if !semver.IsValid(version) {
-		return nil, errors.New("version is not a valid semver: " + version)
-	}
-	if len(images) == 0 {
-		const rancherWindowsImagesFile = "rancher-windows-images.txt"
-		const rancherImagesFile = "rancher-images.txt"
-
-		rancherWindowsImages, err := rancherPrimeArtifact(imagesListURL)
-		if err != nil {
-			return nil, errors.New("failed to get rancher windows images: " + err.Error())
+func GenerateMissingImagesList(imagesListURL, registry string, concurrencyLimit int, checkImages, ignoreImages []string) ([]string, error) {
+	if len(checkImages) == 0 {
+		if imagesListURL == "" {
+			return nil, errors.New("if no images are provided, an images list URL must be provided")
 		}
-
 		rancherImages, err := rancherPrimeArtifact(imagesListURL)
 		if err != nil {
 			return nil, errors.New("failed to get rancher images: " + err.Error())
 		}
+		checkImages = append(checkImages, rancherImages...)
+	}
 
-		images = append(rancherWindowsImages, rancherImages...)
+	ignore, err := imageSliceToMap(ignoreImages)
+	if err != nil {
+		return nil, err
 	}
 
 	// create an error group with a limit to prevent accidentaly doing a DOS attack against our registry
 	ctx, cancel := context.WithCancel(context.Background())
 	errGroup, ctx := errgroup.WithContext(ctx)
 	errGroup.SetLimit(concurrencyLimit)
-	missingImagesChan := make(chan string, len(images))
+	missingImagesChan := make(chan string, len(checkImages))
 
 	// auth tokens can be reused, but maps need a lock for reading and writing in go routines
 	repositoryAuths := make(map[string]string)
 	mu := sync.RWMutex{}
 
-	for _, imageAndVersion := range images {
-		if !strings.Contains(imageAndVersion, ":") {
+	for _, imageAndVersion := range checkImages {
+		image, imageVersion, err := splitImageAndVersion(imageAndVersion)
+		if err != nil {
 			cancel()
-			return nil, errors.New("malformed image name: , missing ':'")
+			return nil, err
 		}
 
-		splitImage := strings.Split(imageAndVersion, ":")
-		image := splitImage[0]
-		imageVersion := splitImage[1]
+		if _, ok := ignore[image]; ok {
+			log.Println("skipping ignored image: " + imageAndVersion)
+			continue
+		}
 
 		func(ctx context.Context, missingImagesChan chan string, image, imageVersion string, repositoryAuths map[string]string, mu *sync.RWMutex) {
 			errGroup.Go(func() error {
@@ -484,6 +482,45 @@ func GenerateMissingImagesList(version, imagesListURL string, concurrencyLimit i
 	return missingImages, nil
 }
 
+func imageSliceToMap(images []string) (map[string]bool, error) {
+	imagesMap := make(map[string]bool, len(images))
+	for _, image := range images {
+		if err := validateRepoImage(image); err != nil {
+			return nil, err
+		}
+		imagesMap[image] = true
+	}
+	return imagesMap, nil
+}
+
+// splitImageAndVersion will validate the image format and return
+// repo/image, version and any validation errors
+// e.g: rancher/rancher-agent:v2.9.0
+func splitImageAndVersion(image string) (string, string, error) {
+	if !strings.Contains(image, ":") {
+		return "", "", errors.New("malformed image name, missing ':' " + image)
+	}
+	splitImage := strings.Split(image, ":")
+	repoImage := splitImage[0]
+	if err := validateRepoImage(repoImage); err != nil {
+		return "", "", err
+	}
+	imageVersion := splitImage[1]
+	return repoImage, imageVersion, nil
+}
+
+// validateRepoImage will validate that a given string only contains
+// the repo and image names and not the version. e.g: rancher/rancher
+func validateRepoImage(repoImage string) error {
+	if !strings.Contains(repoImage, "/") {
+		return errors.New("malformed image name, missing '/' " + repoImage)
+	}
+	if strings.Contains(repoImage, ":") {
+		return errors.New("malformed image name, the repo and image name shouldn't contain versions: " + repoImage)
+	}
+	return nil
+}
+
 func GenerateDockerImageDigests(outputFile, imagesFileURL, registry string) error {
 	imagesDigests, err := dockerImagesDigests(imagesFileURL, registry)
 	if err != nil {

release/rancher/rancher_test.go

@@ -0,0 +1,60 @@
+package rancher
+
+import "testing"
+
+const (
+	rancherRepoImage = "rancher/rancher"
+	rancherVersion   = "v2.9.0"
+)
+
+var (
+	imagesWithVersion = []string{
+		rancherRepoImage + ":" + rancherVersion,
+		"rancher/rancher-agent:v2.9.0",
+		"k3s-io/k3s:v1.25.4",
+	}
+	imagesWithoutVersion = []string{
+		rancherRepoImage,
+		"rancher/rancher-agent",
+		"k3s-io/k3s",
+	}
+)
+
+func TestImageSliceToMap(t *testing.T) {
+	images, err := imageSliceToMap(imagesWithoutVersion)
+	if err != nil {
+		t.Error(err)
+	}
+	if _, ok := images[imagesWithoutVersion[0]]; !ok {
+		t.Error("expected image not found on map " + imagesWithoutVersion[0])
+	}
+	images, err = imageSliceToMap(imagesWithVersion)
+	if err == nil {
+		t.Error("expected to flag image with version as malformed")
+	}
+}
+
+func TestValidateRepoImage(t *testing.T) {
+	if err := validateRepoImage(rancherRepoImage); err != nil {
+		t.Error(err)
+	}
+	if err := validateRepoImage(imagesWithVersion[0]); err == nil {
+		t.Error("expected to flag image with version as malformed" + imagesWithVersion[0])
+	}
+}
+
+func TestSplitImageAndVersion(t *testing.T) {
+	repoImage, version, err := splitImageAndVersion(imagesWithVersion[0])
+	if err != nil {
+		t.Error(err)
+	}
+	if repoImage != rancherRepoImage {
+		t.Error("expected repoImage to be " + rancherRepoImage + " instead, got " + repoImage)
+	}
+	if version != rancherVersion {
+		t.Error("expected version to be " + rancherVersion + " instead, got " + version)
+	}
+	if _, _, err := splitImageAndVersion(imagesWithoutVersion[0]); err == nil {
+		t.Error("expected to flag image without version as malformed " + imagesWithoutVersion[0])
+	}
+}