plume: add cosa2stream

Part of implementing openshift/os#477

For now I decided to go directly from cosa to a stream because:

 - The cosa2release code is in a separate repo in Python
 - The RHCOS pipeline currently generates "one big build" anyways

mantle/cmd/plume/cosa2stream.go

@@ -0,0 +1,195 @@
+// Copyright Red Hat, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/coreos/mantle/cosa"
+	"github.com/coreos/stream-metadata-go/stream"
+	"github.com/spf13/cobra"
+)
+
+var (
+	cmdCosaBuildToStream = &cobra.Command{
+		Use:   "cosa2stream [options]",
+		Short: "Generate stream JSON from a coreos-assembler build",
+		RunE:  runCosaBuildToStream,
+
+		SilenceUsage: true,
+	}
+
+	streamBaseURL string
+	streamName    string
+)
+
+func init() {
+	cmdCosaBuildToStream.Flags().StringVar(&streamBaseURL, "url", "", "Base URL for build")
+	cmdCosaBuildToStream.Flags().StringVar(&streamName, "name", "", "Stream name")
+	cmdCosaBuildToStream.MarkFlagRequired("name")
+	root.AddCommand(cmdCosaBuildToStream)
+}
+
+func getExtension(path string, ext string) (string, error) {
+	ext = "." + ext
+	i := strings.LastIndex(path, ext)
+	if i == -1 {
+		return "", fmt.Errorf("Path %s does not match extension %s", path, ext)
+	}
+	return path[i+1:], nil
+}
+
+func mapArtifact(ca *cosa.Artifact, url string) *stream.Artifact {
+	return &stream.Artifact{
+		Location: url + ca.Path,
+		Sha256:   ca.Sha256,
+	}
+}
+
+// extendStreamFromCosaBuild appends the contents of a cosa build to
+// the provided stream.
+func extendStreamFromCosaBuild(s *stream.Stream, build *cosa.Build, url string) error {
+	sa := s.Architectures[build.Architecture]
+	if sa.Artifacts == nil {
+		sa.Artifacts = make(map[string]stream.PlatformArtifacts)
+	}
+
+	if build.BuildArtifacts == nil {
+		fmt.Fprintf(os.Stderr, "Skipping build %s/%s: missing artifacts\n", build.BuildID, build.Architecture)
+		return nil
+	}
+	cosaArtifacts := build.BuildArtifacts
+
+	if cosaArtifacts.Qemu != nil {
+		artifacts := stream.PlatformArtifacts{
+			Release: build.BuildID,
+			Formats: make(map[string]stream.ImageFormat),
+		}
+		ext, err := getExtension(cosaArtifacts.Qemu.Path, "qcow2")
+		if err != nil {
+			return err
+		}
+		artifacts.Formats[ext] = stream.ImageFormat{
+			Disk: mapArtifact(cosaArtifacts.Qemu, url),
+		}
+		sa.Artifacts["qemu"] = artifacts
+	}
+
+	// Special cloud types (aws/gcp)
+	if len(build.Amis) > 0 {
+		regions := make(map[string]stream.AwsRegionImage)
+		for _, ami := range build.Amis {
+			regions[ami.Region] = stream.AwsRegionImage{
+				Release: build.BuildID,
+				Image:   ami.Hvm,
+			}
+		}
+		sa.Images = stream.Images{
+			Aws: &stream.AwsImage{
+				Regions: regions,
+			},
+		}
+	}
+	s.Architectures[build.Architecture] = sa
+	return nil
+}
+
+type fetchedCosaBuild struct {
+	build *cosa.Build
+	url   string
+}
+
+// parseLocalOrRemoteBuild loads cosa builds from a local directory
+// or a URL.
+func parseLocalOrRemoteBuild(arg string) ([]fetchedCosaBuild, error) {
+	// If it looks like a local directory, iterate over all architecture
+	// subdirectories.
+	if strings.HasPrefix(arg, "./") {
+		builds := []fetchedCosaBuild{}
+		ents, err := ioutil.ReadDir(arg)
+		if err != nil {
+			return nil, err
+		}
+		for _, ent := range ents {
+			metap := filepath.Join(arg, ent.Name(), "meta.json")
+			if _, err := os.Stat(metap); err != nil {
+				continue
+			}
+			build, err := cosa.ParseBuild(metap)
+			if err != nil {
+				return nil, err
+			}
+			builds = append(builds, fetchedCosaBuild{
+				build: build,
+				url:   ""})
+		}
+		if len(builds) == 0 {
+			fmt.Fprintf(os.Stderr, "warning: No builds found in %s\n", arg)
+		}
+		return builds, nil
+	}
+	// URL case
+	build, err := cosa.FetchAndParseBuild(arg)
+	if err != nil {
+		return nil, err
+	}
+	baseurl := filepath.Dir(arg) + "/"
+	return []fetchedCosaBuild{fetchedCosaBuild{build: build, url: baseurl}}, nil
+}
+
+func runCosaBuildToStream(cmd *cobra.Command, args []string) error {
+	// Canonicalize the URL
+	if !strings.HasSuffix(streamBaseURL, "/") {
+		streamBaseURL += "/"
+	}
+
+	// Generate output stream skeleton
+	outStream := stream.Stream{
+		Stream:        streamName,
+		Metadata:      stream.Metadata{LastModified: time.Now().UTC().Format(time.RFC3339)},
+		Architectures: make(map[string]stream.Arch),
+	}
+
+	// Gather all input cosa builds
+	builds := []fetchedCosaBuild{}
+	for _, arg := range args {
+		parsedBuilds, err := parseLocalOrRemoteBuild(arg)
+		if err != nil {
+			return err
+		}
+		builds = append(builds, parsedBuilds...)
+	}
+
+	// Extend the stream using each build as input
+	for _, build := range builds {
+		err := extendStreamFromCosaBuild(&outStream, build.build, build.url)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Serialize to JSON
+	encoder := json.NewEncoder(os.Stdout)
+	if err := encoder.Encode(&outStream); err != nil {
+		return fmt.Errorf("Error while encoding: %v", err)
+	}
+	return nil
+}

mantle/go.sum

@@ -105,6 +105,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/stream-metadata-go v0.0.0-20210107232620-d808ce9d237c h1:7VO10dpKljeaYJUQtObhqjNxpuTCUDELTviJsGy9OeM=
 github.com/coreos/stream-metadata-go v0.0.0-20210107232620-d808ce9d237c/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko=
+github.com/coreos/stream-metadata-go v0.0.0-20210112152733-52b38c241a3d h1:a65dhEcT+kL9Bf5pDpdoOMdT5w0VjwUXE+XO2MoFuSg=
 github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068 h1:y2aHj7QqyAJ6YBBONTAr17YxHHiogDkYnTsJvFNhxwY=
 github.com/coreos/vcontext v0.0.0-20190529201340-22b159166068/go.mod h1:E+6hug9bFSe0KZ2ZAzr8M9F5JlArJjv5D1JS7KSkPKE=
 github.com/coreos/vcontext v0.0.0-20201120045928-b0e13dab675c h1:jA28WeORitsxGFVWhyWB06sAG2HbLHPQuHwDydhU2CQ=