plume prune enhancement

cmd/plume/prune.go

@@ -18,10 +18,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/coreos/pkg/capnslog"
 	"github.com/spf13/cobra"
 	"golang.org/x/net/context"
 
@@ -30,9 +32,13 @@ import (
 )
 
 var (
-	days        int
-	pruneDryRun bool
-	cmdPrune    = &cobra.Command{
+	days              int
+	daysSoftDeleted   int
+	daysLastLaunched  int
+	keepLast          int
+	pruneDryRun       bool
+	checkLastLaunched bool
+	cmdPrune          = &cobra.Command{
 		Use:   "prune --channel CHANNEL [options]",
 		Short: "Prune old release images for the given channel.",
 		Run:   runPrune,
@@ -42,12 +48,17 @@ var (
 
 func init() {
 	cmdPrune.Flags().IntVar(&days, "days", 30, "Minimum age in days for files to get deleted")
+	cmdPrune.Flags().IntVar(&daysLastLaunched, "days-last-launched", 0,
+		"Minimum lastLaunchedTime value in days for images to be deleted. Only used when --check-last-launched is set. If not provided, --days value is used.")
+	cmdPrune.Flags().IntVar(&daysSoftDeleted, "days-soft-deleted", 0, "Minimum age in days for files to remain soft deleted (recoverable)")
+	cmdPrune.Flags().IntVar(&keepLast, "keep-last", 0, "Number of latest images to keep")
 	cmdPrune.Flags().StringVar(&awsCredentialsFile, "aws-credentials", "", "AWS credentials file")
 	cmdPrune.Flags().StringVar(&azureProfile, "azure-profile", "", "Azure Profile json file")
 	cmdPrune.Flags().StringVar(&azureAuth, "azure-auth", "", "Azure Credentials json file")
 	cmdPrune.Flags().StringVar(&azureTestContainer, "azure-test-container", "", "Use another container instead of the default")
 	cmdPrune.Flags().BoolVarP(&pruneDryRun, "dry-run", "n", false,
 		"perform a trial run, do not make changes")
+	cmdPrune.Flags().BoolVarP(&checkLastLaunched, "check-last-launched", "c", false, "Check whether image has been launched recently")
 	AddSpecFlags(cmdPrune.Flags())
 	root.AddCommand(cmdPrune)
 }
@@ -56,6 +67,21 @@ func runPrune(cmd *cobra.Command, args []string) {
 	if len(args) > 0 {
 		plog.Fatal("No args accepted")
 	}
+	if daysLastLaunched < 0 {
+		plog.Fatal("days-last-launched must be >= 0")
+	}
+	if daysSoftDeleted < 0 {
+		plog.Fatal("days-soft-deleted must be >= 0")
+	}
+	if keepLast < 0 {
+		plog.Fatal("keep-last must be >= 0")
+	}
+	if !checkLastLaunched && daysLastLaunched > 0 {
+		plog.Fatal("days-last-launched is ignored when check-last-launched is not set")
+	}
+	if checkLastLaunched && daysLastLaunched == 0 {
+		daysLastLaunched = days
+	}
 
 	// Override specVersion as it's not relevant for this command
 	specVersion = "none"
@@ -151,16 +177,27 @@ func pruneAzure(ctx context.Context, spec *channelSpec) {
 	}
 }
 
+type deleteStats struct {
+	total        int
+	kept         int
+	skipped      int
+	recentlyUsed int
+	softDeleted  int
+	deleted      int
+}
+
 func pruneAWS(ctx context.Context, spec *channelSpec) {
 	if spec.AWS.Image == "" || awsCredentialsFile == "" {
 		plog.Notice("AWS image pruning disabled.")
 		return
 	}
+	stats := deleteStats{}
 
 	// Iterate over all partitions and regions in the given channel and prune
 	// images in each of them.
 	for _, part := range spec.AWS.Partitions {
 		for _, region := range part.Regions {
+			plog := capnslog.NewPackageLogger("github.com/flatcar-linux/mantle", fmt.Sprintf("prune:%s", region))
 			if pruneDryRun {
 				plog.Printf("Checking for images in %v...", part.Name)
 			} else {
@@ -180,9 +217,27 @@ func pruneAWS(ctx context.Context, spec *channelSpec) {
 			if err != nil {
 				plog.Fatalf("Couldn't list images in channel %q: %v", specChannel, err)
 			}
+			stats.total += len(images)
 
 			plog.Infof("Got %d images with channel %q", len(images), specChannel)
 
+			// sort images by creation date
+			sort.Slice(images, func(i, j int) bool {
+				datei, _ := time.Parse(time.RFC3339Nano, *images[i].CreationDate)
+				datej, _ := time.Parse(time.RFC3339Nano, *images[j].CreationDate)
+				return datei.Before(datej)
+			})
+			if len(images) <= keepLast {
+				plog.Infof("Not enough images to prune, keeping %d", len(images))
+				stats.kept += len(images)
+				continue
+			}
+			for _, image := range images[len(images)-keepLast:] {
+				plog.Infof("Keeping image %q", *image.Name)
+			}
+			stats.kept += keepLast
+			images = images[:len(images)-keepLast]
+
 			now := time.Now()
 			for _, image := range images {
 				creationDate, err := time.Parse(time.RFC3339Nano, *image.CreationDate)
@@ -193,9 +248,24 @@ func pruneAWS(ctx context.Context, spec *channelSpec) {
 				daysOld := int(duration.Hours() / 24)
 				if daysOld < days {
 					plog.Infof("Valid image %q: %d days old, skipping", *image.Name, daysOld)
+					stats.skipped += 1
 					continue
 				}
-				plog.Infof("Obsolete image %q: %d days old", *image.Name, daysOld)
+				if checkLastLaunched {
+					lastLaunched, err := api.GetImageLastLaunchedTime(*image.ImageId)
+					if err != nil {
+						plog.Warningf("Error converting last launched date (%v): %v", *image.ImageId, err)
+						continue
+					}
+					duration := now.Sub(lastLaunched)
+					daysOld := int(duration.Hours() / 24)
+					if daysOld < daysLastLaunched {
+						plog.Infof("Image %q: recently used %d days ago (%v), skipping", *image.Name, daysOld, lastLaunched)
+						stats.recentlyUsed += 1
+						continue
+					}
+				}
+				plog.Infof("Obsolete image %q/%q: %d days old", *image.Name, *image.ImageId, daysOld)
 				if !pruneDryRun {
 					// Construct the s3ObjectPath in the same manner it's constructed for upload
 					arch := *image.Architecture
@@ -204,11 +274,45 @@ func pruneAWS(ctx context.Context, spec *channelSpec) {
 					}
 					board := fmt.Sprintf("%s-usr", arch)
 					var version string
+					var softDeleteDate string
 					for _, t := range image.Tags {
 						if *t.Key == "Version" {
 							version = *t.Value
 						}
+						if *t.Key == "SoftDeleteDate" {
+							softDeleteDate = *t.Value
+						}
 					}
+					if softDeleteDate == "" && daysSoftDeleted > 0 {
+						softDeleteDate = now.Format(time.RFC3339)
+						// remove LaunchPermission
+						_, err = api.RemoveLaunchPermission(*image.ImageId)
+						if err != nil {
+							plog.Fatalf("Error removing launch permission from %v: %v", *image.Name, err)
+						}
+						// add tag
+						err = api.CreateTags([]string{*image.ImageId}, map[string]string{"SoftDeleteDate": softDeleteDate})
+						if err != nil {
+							plog.Fatalf("Error adding tag to %v: %v", *image.Name, err)
+						}
+						plog.Infof("Image %v has been soft-deleted", *image.Name)
+						stats.softDeleted += 1
+						continue
+					} else if daysSoftDeleted > 0 {
+						// check if the image is still soft-deleted
+						softDeleteDateTs, err := time.Parse(time.RFC3339, softDeleteDate)
+						if err != nil {
+							plog.Fatalf("Error converting soft-delete date (%v): %v", softDeleteDateTs, err)
+						}
+						duration := now.Sub(softDeleteDateTs)
+						daysOld := int(duration.Hours() / 24)
+						if daysOld < daysSoftDeleted {
+							plog.Infof("Image %v soft-deleted %d days ago, skipping", *image.Name, daysOld)
+							stats.softDeleted += 1
+							continue
+						}
+					}
+
 					imageFileName := strings.TrimSuffix(spec.AWS.Image, filepath.Ext(spec.AWS.Image))
 					s3ObjectPath := fmt.Sprintf("%s/%s/%s", board, version, imageFileName)
 
@@ -219,8 +323,10 @@ func pruneAWS(ctx context.Context, spec *channelSpec) {
 					if err != nil {
 						plog.Fatalf("couldn't prune image %v: %v", *image.Name, err)
 					}
+					stats.deleted += 1
 				}
 			}
 		}
 	}
+	plog.Noticef("Pruning complete: %+v", stats)
 }

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Azure/azure-sdk-for-go v56.2.0+incompatible
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8
 	github.com/Microsoft/azure-vhd-utils v0.0.0-20210818134022-97083698b75f
-	github.com/aws/aws-sdk-go v1.42.41
+	github.com/aws/aws-sdk-go v1.44.46
 	github.com/coreos/butane v0.14.1-0.20220401164106-6b5239299226
 	github.com/coreos/coreos-cloudinit v1.11.0
 	github.com/coreos/go-iptables v0.5.0

go.sum

@@ -104,8 +104,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
 github.com/aws/aws-sdk-go v1.30.28/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.42.41 h1:gmHgDzSiLYfHx0ZedcXtHjMXbVxBtyOcl/sy0wg9o2o=
-github.com/aws/aws-sdk-go v1.42.41/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc=
+github.com/aws/aws-sdk-go v1.44.46 h1:BsKENvu24eXg7CWQ2wJAjKbDFkGP+hBtxKJIR3UdcB8=
+github.com/aws/aws-sdk-go v1.44.46/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/beevik/etree v1.1.1-0.20200718192613-4a2f8b9d084c/go.mod h1:0yGO2rna3S9DkITDWHY1bMtcY4IJ4w+4S+EooZUR0bE=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -683,7 +683,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc=

platform/api/aws/images.go

@@ -15,6 +15,7 @@
 package aws
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/url"
 	"strings"
@@ -333,6 +334,28 @@ func (a *API) CreateHVMImage(snapshotID string, diskSizeGiB uint, name string, d
 	return a.createImage(params)
 }
 
+func (a *API) RemoveLaunchPermission(imageid string) ([]byte, error) {
+	mod := &ec2.ModifyImageAttributeInput{
+		ImageId: &imageid,
+		LaunchPermission: &ec2.LaunchPermissionModifications{
+			Remove: []*ec2.LaunchPermission{
+				{
+					Group: aws.String("all"),
+				},
+			},
+		},
+	}
+	resp, err := a.ec2.ModifyImageAttribute(mod)
+	if err != nil {
+		return []byte{}, fmt.Errorf("modifying image attribute: %w", err)
+	}
+	out, err := json.Marshal(resp)
+	if err != nil {
+		return []byte{}, fmt.Errorf("marshaling the API response: %w", err)
+	}
+	return out, nil
+}
+
 func (a *API) deregisterImageIfExists(name string) error {
 	imageID, err := a.FindImage(name)
 	if err != nil {
@@ -676,6 +699,24 @@ func (a *API) describeImage(imageID string) (*ec2.Image, error) {
 	return describeRes.Images[0], nil
 }
 
+func (a *API) GetImageLastLaunchedTime(imageID string) (time.Time, error) {
+	resp, err := a.ec2.DescribeImageAttribute(&ec2.DescribeImageAttributeInput{
+		Attribute: aws.String(ec2.ImageAttributeNameLastLaunchedTime),
+		ImageId:   aws.String(imageID),
+	})
+	if err != nil {
+		return time.Time{}, err
+	}
+	if resp.LastLaunchedTime == nil || resp.LastLaunchedTime.Value == nil {
+		return time.Time{}, nil
+	}
+	lastLaunchedTime, err := time.Parse(time.RFC3339Nano, *resp.LastLaunchedTime.Value)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return lastLaunchedTime, nil
+}
+
 // GetImagesByTag returns all EC2 images with that tag
 func (a *API) GetImagesByTag(tag, value string) ([]*ec2.Image, error) {
 	describeRes, err := a.ec2.DescribeImages(&ec2.DescribeImagesInput{