[WIP] MGMT-2977 Create cluster-specific minimal iso

cmd/main.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -154,6 +155,8 @@ func main() {
 
 	log.Println(fmt.Sprintf("Started service with OCP versions %v", openshiftVersionsMap))
 
+	failOnError(os.MkdirAll(Options.BMConfig.ISOCacheDir, 0700), "Failed to create ISO cache directory %s", Options.BMConfig.ISOCacheDir)
+
 	// Connect to db
 	db := setupDB(log)
 	defer db.Close()

internal/bminventory/inventory.go

@@ -18,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"regexp"
 	"sort"
 	"strconv"
@@ -42,6 +43,7 @@ import (
 	"github.com/openshift/assisted-service/internal/identity"
 	"github.com/openshift/assisted-service/internal/ignition"
 	"github.com/openshift/assisted-service/internal/installcfg"
+	"github.com/openshift/assisted-service/internal/isoeditor"
 	"github.com/openshift/assisted-service/internal/manifests"
 	"github.com/openshift/assisted-service/internal/metrics"
 	"github.com/openshift/assisted-service/internal/network"
@@ -96,6 +98,7 @@ type Config struct {
 	ServiceIPs               string            `envconfig:"SERVICE_IPS" default:""`
 	DeletedUnregisteredAfter time.Duration     `envconfig:"DELETED_UNREGISTERED_AFTER" default:"168h"`
 	DefaultNTPSource         string            `envconfig:"NTP_DEFAULT_SERVER"`
+	ISOCacheDir              string            `envconfig:"ISO_CACHE_DIR" default:"/tmp/isocache"`
 }
 
 const agentMessageOfTheDay = `
@@ -833,7 +836,7 @@ func (b *bareMetalInventory) DownloadClusterISO(ctx context.Context, params inst
 		contentLength)
 }
 
-func (b *bareMetalInventory) updateImageInfoPostUpload(ctx context.Context, cluster *common.Cluster, clusterProxyHash string) error {
+func (b *bareMetalInventory) updateImageInfoPostUpload(ctx context.Context, cluster *common.Cluster, clusterProxyHash string, imageType models.ImageType) error {
 	updates := map[string]interface{}{}
 	imgName := getImageName(*cluster.ID)
 	imgSize, err := b.objectHandler.GetObjectSizeBytes(ctx, imgName)
@@ -858,6 +861,9 @@ func (b *bareMetalInventory) updateImageInfoPostUpload(ctx context.Context, clus
 		cluster.ProxyHash = clusterProxyHash
 	}
 
+	updates["image_type"] = imageType
+	cluster.ImageInfo.Type = imageType
+
 	dbReply := b.db.Model(&common.Cluster{}).Where("id = ?", cluster.ID.String()).Updates(updates)
 	if dbReply.Error != nil {
 		return errors.New("Failed to generate image: error updating image record")
@@ -940,7 +946,8 @@ func (b *bareMetalInventory) GenerateClusterISOInternal(ctx context.Context, par
 	var imageExists bool
 	if cluster.ImageInfo.SSHPublicKey == params.ImageCreateParams.SSHPublicKey &&
 		cluster.ProxyHash == clusterProxyHash &&
-		cluster.ImageInfo.StaticIpsConfig == staticIpsConfig {
+		cluster.ImageInfo.StaticIpsConfig == staticIpsConfig &&
+		cluster.ImageInfo.Type == params.ImageCreateParams.ImageType {
 		var err error
 		imgName := getImageName(params.ClusterID)
 		imageExists, err = b.objectHandler.UpdateObjectTimestamp(ctx, imgName)
@@ -981,7 +988,7 @@ func (b *bareMetalInventory) GenerateClusterISOInternal(ctx context.Context, par
 	}
 
 	if imageExists {
-		if err := b.updateImageInfoPostUpload(ctx, &cluster, clusterProxyHash); err != nil {
+		if err := b.updateImageInfoPostUpload(ctx, &cluster, clusterProxyHash, params.ImageCreateParams.ImageType); err != nil {
 			return nil, common.NewApiError(http.StatusInternalServerError, err)
 		}
 
@@ -1002,14 +1009,41 @@ func (b *bareMetalInventory) GenerateClusterISOInternal(ctx context.Context, par
 		return nil, common.NewApiError(http.StatusInternalServerError, err)
 	}
 
-	if err := b.objectHandler.UploadISO(ctx, ignitionConfig, b.objectHandler.GetBaseIsoObject(cluster.OpenshiftVersion),
-		fmt.Sprintf(s3wrapper.DiscoveryImageTemplate, cluster.ID.String())); err != nil {
-		log.WithError(err).Errorf("Upload ISO failed for cluster %s", cluster.ID)
-		b.eventsHandler.AddEvent(ctx, params.ClusterID, nil, models.EventSeverityError, "Failed to upload image", time.Now())
-		return nil, common.NewApiError(http.StatusInternalServerError, err)
+	objectPrefix := fmt.Sprintf(s3wrapper.DiscoveryImageTemplate, cluster.ID.String())
+
+	if params.ImageCreateParams.ImageType == models.ImageTypeMinimalIso {
+		baseISOName := s3wrapper.GetMinimalIsoObjectName(cluster.OpenshiftVersion)
+		isoPath, err := s3wrapper.GetFile(ctx, b.objectHandler, baseISOName, b.ISOCacheDir, true)
+		if err != nil {
+			log.WithError(err).Errorf("Failed to download minimal ISO template %s", baseISOName)
+			return nil, common.NewApiError(http.StatusInternalServerError, err)
+		}
+
+		log.Infof("Creating minimal ISO for cluster %s", cluster.ID)
+		clusterISOPath, err := isoeditor.CreateEditor(isoPath, cluster.OpenshiftVersion, log).CreateClusterMinimalISO(ignitionConfig)
+		if err != nil {
+			log.WithError(err).Errorf("Failed to create minimal discovery ISO for cluster %s", cluster.ID)
+			return nil, common.NewApiError(http.StatusInternalServerError, err)
+		}
+
+		log.Infof("Uploading minimal ISO for cluster %s", cluster.ID)
+		if err := b.objectHandler.UploadFile(ctx, clusterISOPath, fmt.Sprintf("%s.iso", objectPrefix)); err != nil {
+			os.Remove(clusterISOPath)
+			log.WithError(err).Errorf("Failed to upload minimal discovery ISO for cluster %s", cluster.ID)
+			return nil, common.NewApiError(http.StatusInternalServerError, err)
+		}
+		os.Remove(clusterISOPath)
+	} else {
+		baseISOName := b.objectHandler.GetBaseIsoObject(cluster.OpenshiftVersion)
+
+		if err := b.objectHandler.UploadISO(ctx, ignitionConfig, baseISOName, objectPrefix); err != nil {
+			log.WithError(err).Errorf("Upload ISO failed for cluster %s", cluster.ID)
+			b.eventsHandler.AddEvent(ctx, params.ClusterID, nil, models.EventSeverityError, "Failed to upload image", time.Now())
+			return nil, common.NewApiError(http.StatusInternalServerError, err)
+		}
 	}
 
-	if err := b.updateImageInfoPostUpload(ctx, &cluster, clusterProxyHash); err != nil {
+	if err := b.updateImageInfoPostUpload(ctx, &cluster, clusterProxyHash, params.ImageCreateParams.ImageType); err != nil {
 		return nil, common.NewApiError(http.StatusInternalServerError, err)
 	}
 

internal/isoeditor/rhcos.go

@@ -1,14 +1,24 @@
 package isoeditor
 
 import (
+	"bytes"
+	"compress/gzip"
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"regexp"
 
 	"github.com/openshift/assisted-service/internal/isoutil"
 	"github.com/openshift/assisted-service/restapi/operations/bootfiles"
+
+	"github.com/cavaliercoder/go-cpio"
+	config_31 "github.com/coreos/ignition/v2/config/v3_1"
+	config_31_types "github.com/coreos/ignition/v2/config/v3_1/types"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"github.com/thoas/go-funk"
+	"github.com/vincent-petithory/dataurl"
 )
 
 const (
@@ -18,6 +28,7 @@ const (
 
 type Editor interface {
 	CreateMinimalISOTemplate(serviceBaseURL string) (string, error)
+	CreateClusterMinimalISO(ignition string) (string, error)
 }
 
 type rhcosEditor struct {
@@ -77,31 +88,159 @@ func (e *rhcosEditor) CreateMinimalISOTemplate(serviceBaseURL string) (string, e
 		return "", err
 	}
 
-	isoPath, err := tempFileName()
-	if err != nil {
+	e.log.Info("Creating minimal ISO template")
+	return e.create()
+}
+
+// CreateClusterMinimalISO creates a new rhcos iso with cluser file customizations added
+// to the initrd image
+func (e *rhcosEditor) CreateClusterMinimalISO(ignition string) (string, error) {
+	if err := e.isoHandler.Extract(); err != nil {
+		return "", err
+	}
+	defer func() {
+		if err := e.isoHandler.CleanWorkDir(); err != nil {
+			e.log.WithError(err).Warnf("Failed to clean isoHandler work dir")
+		}
+	}()
+
+	if err := e.addIgnitionFiles(ignition); err != nil {
 		return "", err
 	}
 
-	e.log.Infof("Creating minimal ISO template: %s", isoPath)
-	if err := e.create(isoPath); err != nil {
+	if err := e.addIgnitionArchive(ignition); err != nil {
 		return "", err
 	}
 
-	return isoPath, nil
+	return e.create()
 }
 
-func (e *rhcosEditor) create(outPath string) error {
-	volumeID, err := e.isoHandler.VolumeIdentifier()
+func (e *rhcosEditor) addIgnitionArchive(ignition string) error {
+	archiveBytes, err := IgnitionImageArchive(ignition)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(e.isoHandler.ExtractedPath("images/ignition.img"), archiveBytes, 0644)
+}
+
+func addFile(w *cpio.Writer, f config_31_types.File) error {
+	u, err := dataurl.DecodeString(f.Contents.Key())
 	if err != nil {
 		return err
 	}
-	if err = e.isoHandler.Create(outPath, volumeID); err != nil {
+
+	var mode cpio.FileMode = 0644
+	if f.Mode != nil {
+		mode = cpio.FileMode(*f.Mode)
+	}
+
+	uid := 0
+	if f.User.ID != nil {
+		uid = *f.User.ID
+	}
+
+	gid := 0
+	if f.Group.ID != nil {
+		gid = *f.Group.ID
+	}
+
+	// add the file
+	hdr := &cpio.Header{
+		Name: f.Path,
+		Mode: mode,
+		UID:  uid,
+		GID:  gid,
+		Size: int64(len(u.Data)),
+	}
+	if err := w.WriteHeader(hdr); err != nil {
+		return err
+	}
+	if _, err := w.Write(u.Data); err != nil {
 		return err
 	}
 
 	return nil
 }
 
+// addIgnitionFiles adds all files referenced in the given ignition config to
+// the initrd by creating an additional cpio archive
+func (e *rhcosEditor) addIgnitionFiles(ignition string) error {
+	config, _, err := config_31.Parse([]byte(ignition))
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Create(e.isoHandler.ExtractedPath("images/assisted_custom_files.img"))
+	if err != nil {
+		return fmt.Errorf("failed to open image file: %s", err)
+	}
+
+	w := cpio.NewWriter(f)
+	addedPaths := make([]string, 0)
+
+	// TODO: deal with config.Storage.Directories also?
+	for _, f := range config.Storage.Files {
+		if err = addFile(w, f); err != nil {
+			return fmt.Errorf("failed to add file %s to archive: %v", f.Path, err)
+		}
+
+		// Need to add all directories in the file path to ensure it can be created
+		// Many files may be in the same directory so we need to track which directories we add to ensure we only add them once
+		for dir := filepath.Dir(f.Path); dir != "" && dir != "/"; dir = filepath.Dir(dir) {
+			if !funk.Contains(addedPaths, dir) {
+				hdr := &cpio.Header{
+					Name: dir,
+					Mode: 040755,
+					Size: 0,
+				}
+				if err = w.WriteHeader(hdr); err != nil {
+					return err
+				}
+				addedPaths = append(addedPaths, dir)
+			}
+		}
+	}
+
+	if err = w.Close(); err != nil {
+		return err
+	}
+
+	err = editFile(e.isoHandler.ExtractedPath("EFI/redhat/grub.cfg"), ` coreos.liveiso=\S+`, "")
+	if err != nil {
+		return err
+	}
+
+	err = editFile(e.isoHandler.ExtractedPath("isolinux/isolinux.cfg"), ` coreos.liveiso=\S+`, "")
+	if err != nil {
+		return err
+	}
+
+	// edit configs to add new image
+	err = editFile(e.isoHandler.ExtractedPath("EFI/redhat/grub.cfg"), `(?m)^(\s+initrd) (.+| )+$`, "$1 $2 /images/assisted_custom_files.img")
+	if err != nil {
+		return err
+	}
+	return editFile(e.isoHandler.ExtractedPath("isolinux/isolinux.cfg"), `(?m)^(\s+append.*initrd=\S+) (.*)$`, "${1},/images/assisted_custom_files.img ${2}")
+}
+
+func (e *rhcosEditor) create() (string, error) {
+	isoPath, err := tempFileName()
+	if err != nil {
+		return "", err
+	}
+
+	volumeID, err := e.isoHandler.VolumeIdentifier()
+	if err != nil {
+		return "", err
+	}
+	if err = e.isoHandler.Create(isoPath, volumeID); err != nil {
+		return "", err
+	}
+
+	return isoPath, nil
+}
+
 func (e *rhcosEditor) addRootFSURL(serviceBaseURL string) error {
 	replacement := fmt.Sprintf("$1 $2 coreos.live.rootfs_url=%s", e.getRootFSURL(serviceBaseURL))
 	if err := editFile(e.isoHandler.ExtractedPath("EFI/redhat/grub.cfg"), `(?m)^(\s+linux) (.+| )+$`, replacement); err != nil {
@@ -143,3 +282,33 @@ func tempFileName() (string, error) {
 
 	return path, nil
 }
+
+func IgnitionImageArchive(ignitionConfig string) ([]byte, error) {
+	ignitionBytes := []byte(ignitionConfig)
+
+	// Create CPIO archive
+	archiveBuffer := new(bytes.Buffer)
+	cpioWriter := cpio.NewWriter(archiveBuffer)
+	if err := cpioWriter.WriteHeader(&cpio.Header{Name: "config.ign", Mode: 0o100_644, Size: int64(len(ignitionBytes))}); err != nil {
+		return nil, errors.Wrap(err, "Failed to write CPIO header")
+	}
+	if _, err := cpioWriter.Write(ignitionBytes); err != nil {
+
+		return nil, errors.Wrap(err, "Failed to write CPIO archive")
+	}
+	if err := cpioWriter.Close(); err != nil {
+		return nil, errors.Wrap(err, "Failed to close CPIO archive")
+	}
+
+	// Run gzip compression
+	compressedBuffer := new(bytes.Buffer)
+	gzipWriter := gzip.NewWriter(compressedBuffer)
+	if _, err := gzipWriter.Write(archiveBuffer.Bytes()); err != nil {
+		return nil, errors.Wrap(err, "Failed to gzip ignition config")
+	}
+	if err := gzipWriter.Close(); err != nil {
+		return nil, errors.Wrap(err, "Failed to gzip ignition config")
+	}
+
+	return compressedBuffer.Bytes(), nil
+}