diff --git a/cmd/release/cmd/generate.go b/cmd/release/cmd/generate.go index c062dc98..742f5dfa 100644 --- a/cmd/release/cmd/generate.go +++ b/cmd/release/cmd/generate.go @@ -21,6 +21,9 @@ var ( concurrencyLimit int imagesListURL string + ignoreImages []string + checkImages []string + registry string rancherMissingImagesJSONOutput bool rke2PrevMilestone string rke2Milestone string @@ -115,19 +118,13 @@ var rancherGenerateArtifactsIndexSubCmd = &cobra.Command{ } var rancherGenerateMissingImagesListSubCmd = &cobra.Command{ - Use: "missing-images-list [version]", + Use: "missing-images-list", Short: "Generate a missing images list", RunE: func(cmd *cobra.Command, args []string) error { - if len(args) < 1 { - return errors.New("expected at least one argument: [version]") - } - checkImages := make([]string, 0) - version := args[0] - rancherRelease, found := rootConfig.Rancher.Versions[version] - if found { - checkImages = rancherRelease.CheckImages + if len(checkImages) == 0 && imagesListURL == "" { + return errors.New("either --images-list-url or --check-images must be provided") } - missingImages, err := rancher.GenerateMissingImagesList(version, imagesListURL, concurrencyLimit, checkImages) + missingImages, err := rancher.GenerateMissingImagesList(imagesListURL, registry, concurrencyLimit, checkImages, ignoreImages) if err != nil { return err } @@ -204,10 +201,9 @@ func init() { rancherGenerateMissingImagesListSubCmd.Flags().IntVarP(&concurrencyLimit, "concurrency-limit", "l", 3, "Concurrency Limit") rancherGenerateMissingImagesListSubCmd.Flags().BoolVarP(&rancherMissingImagesJSONOutput, "json", "j", false, "JSON Output") rancherGenerateMissingImagesListSubCmd.Flags().StringVarP(&imagesListURL, "images-list-url", "i", "", "URL of the artifact containing all images for a given version 'rancher-images.txt' (required)") - if err := rancherGenerateMissingImagesListSubCmd.MarkFlagRequired("images-list-url"); err != nil { - fmt.Println(err.Error()) - os.Exit(1) - } + rancherGenerateMissingImagesListSubCmd.Flags().StringSliceVarP(&ignoreImages, "ignore-images", "g", make([]string, 0), "Images to ignore when checking for missing images without the version. e.g: rancher/rancher") + rancherGenerateMissingImagesListSubCmd.Flags().StringSliceVarP(&checkImages, "check-images", "k", make([]string, 0), "Images to check for when checking for missing images with the version. e.g: rancher/rancher-agent:v2.9.0") + rancherGenerateMissingImagesListSubCmd.Flags().StringVarP(®istry, "registry", "r", "registry.rancher.com", "Registry where the images should be located at") // rancher generate docker-images-digests rancherGenerateDockerImagesDigestsSubCmd.Flags().StringVarP(&rancherImagesDigestsOutputFile, "output-file", "o", "", "Output file with images digests") diff --git a/cmd/release/cmd/root.go b/cmd/release/cmd/root.go index af0f6045..7d3a2c4b 100644 --- a/cmd/release/cmd/root.go +++ b/cmd/release/cmd/root.go @@ -43,7 +43,7 @@ func SetVersion(version string) { func init() { rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "D", false, "Debug") - rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "R", false, "Drun Run") + rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "R", false, "Dry Run") rootCmd.PersistentFlags().BoolVarP(&ignoreValidate, "ignore-validate", "I", false, "Ignore the validate config step") rootCmd.PersistentFlags().StringVarP(&configFile, "config-file", "c", "$HOME/.ecm-distro-tools/config.json", "Path for the config.json file") rootCmd.PersistentFlags().StringVarP(&stringConfig, "config", "C", "", "JSON config string") diff --git a/cmd/release/config/config.go b/cmd/release/config/config.go index 59127793..0f90c992 100644 --- a/cmd/release/config/config.go +++ b/cmd/release/config/config.go @@ -31,15 +31,14 @@ type K3sRelease struct { // RancherRelease type RancherRelease struct { - ReleaseBranch string `json:"release_branch" validate:"required"` - RancherRepoOwner string `json:"rancher_repo_owner" validate:"required"` - IssueNumber string `json:"issue_number" validate:"number"` - CheckImages []string `json:"check_images" validate:"required"` - BaseRegistry string `json:"base_registry" validate:"required,hostname"` - Registry string `json:"registry" validate:"required,hostname"` - PrimeArtifactsBucket string `json:"prime_artifacts_bucket" validate:"required"` - DryRun bool `json:"dry_run"` - SkipStatusCheck bool `json:"skip_status_check"` + ReleaseBranch string `json:"release_branch" validate:"required"` + RancherRepoOwner string `json:"rancher_repo_owner" validate:"required"` + IssueNumber string `json:"issue_number" validate:"number"` + BaseRegistry string `json:"base_registry" validate:"required,hostname"` + Registry string `json:"registry" validate:"required,hostname"` + PrimeArtifactsBucket string `json:"prime_artifacts_bucket" validate:"required"` + DryRun bool `json:"dry_run"` + SkipStatusCheck bool `json:"skip_status_check"` } // RKE2 @@ -165,7 +164,6 @@ func ExampleConfig() (string, error) { DryRun: false, SkipStatusCheck: false, RancherRepoOwner: "rancher", - CheckImages: []string{}, IssueNumber: "1234", BaseRegistry: "stgregistry.suse.com", Registry: "registry.rancher.com", diff --git a/release/rancher/rancher.go b/release/rancher/rancher.go index e3332553..e89b40fb 100644 --- a/release/rancher/rancher.go +++ b/release/rancher/rancher.go @@ -386,46 +386,44 @@ func formatContentLine(line string) string { return strings.TrimSpace(line) } -func GenerateMissingImagesList(version, imagesListURL string, concurrencyLimit int, images []string) ([]string, error) { - if !semver.IsValid(version) { - return nil, errors.New("version is not a valid semver: " + version) - } - if len(images) == 0 { - const rancherWindowsImagesFile = "rancher-windows-images.txt" - const rancherImagesFile = "rancher-images.txt" - - rancherWindowsImages, err := rancherPrimeArtifact(imagesListURL) - if err != nil { - return nil, errors.New("failed to get rancher windows images: " + err.Error()) +func GenerateMissingImagesList(imagesListURL, registry string, concurrencyLimit int, checkImages, ignoreImages []string) ([]string, error) { + if len(checkImages) == 0 { + if imagesListURL == "" { + return nil, errors.New("if no images are provided, an images list URL must be provided") } - rancherImages, err := rancherPrimeArtifact(imagesListURL) if err != nil { return nil, errors.New("failed to get rancher images: " + err.Error()) } + checkImages = append(checkImages, rancherImages...) + } - images = append(rancherWindowsImages, rancherImages...) + ignore, err := imageSliceToMap(ignoreImages) + if err != nil { + return nil, err } // create an error group with a limit to prevent accidentaly doing a DOS attack against our registry ctx, cancel := context.WithCancel(context.Background()) errGroup, ctx := errgroup.WithContext(ctx) errGroup.SetLimit(concurrencyLimit) - missingImagesChan := make(chan string, len(images)) + missingImagesChan := make(chan string, len(checkImages)) // auth tokens can be reused, but maps need a lock for reading and writing in go routines repositoryAuths := make(map[string]string) mu := sync.RWMutex{} - for _, imageAndVersion := range images { - if !strings.Contains(imageAndVersion, ":") { + for _, imageAndVersion := range checkImages { + image, imageVersion, err := splitImageAndVersion(imageAndVersion) + if err != nil { cancel() - return nil, errors.New("malformed image name: , missing ':'") + return nil, err } - splitImage := strings.Split(imageAndVersion, ":") - image := splitImage[0] - imageVersion := splitImage[1] + if _, ok := ignore[image]; ok { + log.Println("skipping ignored image: " + imageAndVersion) + continue + } func(ctx context.Context, missingImagesChan chan string, image, imageVersion string, repositoryAuths map[string]string, mu *sync.RWMutex) { errGroup.Go(func() error { @@ -484,6 +482,45 @@ func GenerateMissingImagesList(version, imagesListURL string, concurrencyLimit i return missingImages, nil } +func imageSliceToMap(images []string) (map[string]bool, error) { + imagesMap := make(map[string]bool, len(images)) + for _, image := range images { + if err := validateRepoImage(image); err != nil { + return nil, err + } + imagesMap[image] = true + } + return imagesMap, nil +} + +// splitImageAndVersion will validate the image format and return +// repo/image, version and any validation errors +// e.g: rancher/rancher-agent:v2.9.0 +func splitImageAndVersion(image string) (string, string, error) { + if !strings.Contains(image, ":") { + return "", "", errors.New("malformed image name, missing ':' " + image) + } + splitImage := strings.Split(image, ":") + repoImage := splitImage[0] + if err := validateRepoImage(repoImage); err != nil { + return "", "", err + } + imageVersion := splitImage[1] + return repoImage, imageVersion, nil +} + +// validateRepoImage will validate that a given string only contains +// the repo and image names and not the version. e.g: rancher/rancher +func validateRepoImage(repoImage string) error { + if !strings.Contains(repoImage, "/") { + return errors.New("malformed image name, missing '/' " + repoImage) + } + if strings.Contains(repoImage, ":") { + return errors.New("malformed image name, the repo and image name shouldn't contain versions: " + repoImage) + } + return nil +} + func GenerateDockerImageDigests(outputFile, imagesFileURL, registry string) error { imagesDigests, err := dockerImagesDigests(imagesFileURL, registry) if err != nil { diff --git a/release/rancher/rancher_test.go b/release/rancher/rancher_test.go new file mode 100644 index 00000000..1045c362 --- /dev/null +++ b/release/rancher/rancher_test.go @@ -0,0 +1,60 @@ +package rancher + +import "testing" + +const ( + rancherRepoImage = "rancher/rancher" + rancherVersion = "v2.9.0" +) + +var ( + imagesWithVersion = []string{ + rancherRepoImage + ":" + rancherVersion, + "rancher/rancher-agent:v2.9.0", + "k3s-io/k3s:v1.25.4", + } + imagesWithoutVersion = []string{ + rancherRepoImage, + "rancher/rancher-agent", + "k3s-io/k3s", + } +) + +func TestImageSliceToMap(t *testing.T) { + images, err := imageSliceToMap(imagesWithoutVersion) + if err != nil { + t.Error(err) + } + if _, ok := images[imagesWithoutVersion[0]]; !ok { + t.Error("expected image not found on map " + imagesWithoutVersion[0]) + } + images, err = imageSliceToMap(imagesWithVersion) + if err == nil { + t.Error("expected to flag image with version as malformed") + } +} + +func TestValidateRepoImage(t *testing.T) { + if err := validateRepoImage(rancherRepoImage); err != nil { + t.Error(err) + } + if err := validateRepoImage(imagesWithVersion[0]); err == nil { + t.Error("expected to flag image with version as malformed" + imagesWithVersion[0]) + } +} + +func TestSplitImageAndVersion(t *testing.T) { + repoImage, version, err := splitImageAndVersion(imagesWithVersion[0]) + if err != nil { + t.Error(err) + } + if repoImage != rancherRepoImage { + t.Error("expected repoImage to be " + rancherRepoImage + " instead, got " + repoImage) + } + if version != rancherVersion { + t.Error("expected version to be " + rancherVersion + " instead, got " + version) + } + if _, _, err := splitImageAndVersion(imagesWithoutVersion[0]); err == nil { + t.Error("expected to flag image without version as malformed " + imagesWithoutVersion[0]) + } +}