Merge pull request microsoft#2029 from ambarve/layer_refactor

Refactor layer management code.

cmd/containerd-shim-runhcs-v1/delete.go

@@ -4,7 +4,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -18,10 +17,8 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/Microsoft/hcsshim/internal/hcs"
-	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/internal/oc"
-	cimlayer "github.com/Microsoft/hcsshim/internal/wclayer/cim"
 	"github.com/Microsoft/hcsshim/internal/winapi"
 )
 
@@ -126,28 +123,8 @@ The delete command will be executed in the container's bundle as its cwd.
 			fmt.Fprintf(os.Stderr, "failed to delete user %q: %v", username, err)
 		}
 
-		// cleanup the layers mounted for the container. We currently only handle cleanup of CimFS
-		// layers here.  First n-1 values should be the image layerFolders (topmost layer being at
-		// index 0) and the last entry should be the scratch layer
-		var layerFolders []string
-		f, err := os.Open(filepath.Join(bundleFlag, layersFile))
-		if err != nil {
-			if !errors.Is(err, os.ErrNotExist) {
-				fmt.Fprintf(os.Stderr, "open layers file: %s", err)
-			}
-		} else {
-			defer f.Close()
-			if err = json.NewDecoder(f).Decode(&layerFolders); err != nil {
-				fmt.Fprintf(os.Stderr, "decode layers json: %s", err)
-			}
-		}
-		if err == nil && cimlayer.IsCimLayer(layerFolders[0]) {
-			scratchLayerFolderPath := layerFolders[len(layerFolders)-1]
-			err = layers.ReleaseCimFSHostLayers(ctx, scratchLayerFolderPath, idFlag)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "cleanup container %q mounts: %s", idFlag, err)
-			}
-		}
+		// TODO(ambarve):
+		// correctly handle cleanup of cimfs layers in case of shim process crash here.
 
 		if data, err := proto.Marshal(&task.DeleteResponse{
 			ExitedAt:   timestamppb.New(time.Now()),

cmd/containerd-shim-runhcs-v1/main.go

@@ -29,7 +29,6 @@ import (
 
 const usage = ``
 const ttrpcAddressEnv = "TTRPC_ADDRESS"
-const layersFile = "layers.json"
 
 // Add a manifest to get proper Windows version detection.
 //go:generate go run github.com/josephspurrier/goversioninfo/cmd/goversioninfo -platform-specific

cmd/containerd-shim-runhcs-v1/pod.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oci"
 	"github.com/Microsoft/hcsshim/internal/uvm"
@@ -122,22 +123,15 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 				return nil, err
 			}
 		case *uvm.OptionsWCOW:
+			var layerFolders []string
+			if s.Windows != nil {
+				layerFolders = s.Windows.LayerFolders
+			}
 			wopts := (opts).(*uvm.OptionsWCOW)
-
-			// In order for the UVM sandbox.vhdx not to collide with the actual
-			// nested Argon sandbox.vhdx we append the \vm folder to the last
-			// entry in the list.
-			layersLen := len(s.Windows.LayerFolders)
-			layers := make([]string, layersLen)
-			copy(layers, s.Windows.LayerFolders)
-
-			vmPath := filepath.Join(layers[layersLen-1], "vm")
-			err := os.MkdirAll(vmPath, 0)
+			wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, req.Rootfs, layerFolders)
 			if err != nil {
 				return nil, err
 			}
-			layers[layersLen-1] = vmPath
-			wopts.LayerFolders = layers
 
 			parent, err = uvm.CreateWCOW(ctx, wopts)
 			if err != nil {

cmd/containerd-shim-runhcs-v1/service_internal.go

@@ -27,13 +27,6 @@ import (
 
 var empty = &emptypb.Empty{}
 
-// TODO(ambarve): Once we can vendor containerd 2.0 in hcsshim, we should directly reference these types from
-// containerd module
-const (
-	LegacyMountType string = "windows-layer"
-	CimFSMountType  string = "CimFS"
-)
-
 // getPod returns the pod this shim is tracking or else returns `nil`. It is the
 // callers responsibility to verify that `s.isSandbox == true` before calling
 // this method.
@@ -123,53 +116,6 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		}
 	}
 
-	var layerFolders []string
-	if spec.Windows != nil {
-		layerFolders = spec.Windows.LayerFolders
-	}
-	if err := validateRootfsAndLayers(req.Rootfs, layerFolders); err != nil {
-		return nil, err
-	}
-
-	// Only work with Windows here.
-	// Parsing of the rootfs mount for Linux containers occurs later.
-	if spec.Linux == nil && len(req.Rootfs) > 0 {
-		// For Windows containers, we work with LayerFolders throughout
-		// much of the creation logic in the shim. If we were given a
-		// rootfs mount, convert it to LayerFolders here.
-		m := req.Rootfs[0]
-		if m.Type != LegacyMountType && m.Type != CimFSMountType {
-			return nil, fmt.Errorf("unsupported Windows mount type: %s", m.Type)
-		} else if m.Type == CimFSMountType && (shimOpts.SandboxIsolation == runhcsopts.Options_HYPERVISOR) {
-			// For CIMFS layers only process isolation is supported right now.
-			return nil, fmt.Errorf("cimfs doesn't support hyperv isolation")
-		}
-
-		source, parentLayerPaths, err := parseLegacyRootfsMount(m)
-		if err != nil {
-			return nil, err
-		}
-
-		// Append the parents
-		spec.Windows.LayerFolders = append(spec.Windows.LayerFolders, parentLayerPaths...)
-		// Append the scratch
-		spec.Windows.LayerFolders = append(spec.Windows.LayerFolders, source)
-
-		if m.Type == CimFSMountType {
-			// write the layers to a file so that it can be used for proper cleanup during shim
-			// delete. We can't write to the config.json as it is read-only for shim.
-			f, err = os.Create(filepath.Join(req.Bundle, layersFile))
-			if err != nil {
-				return nil, err
-			}
-			if err := json.NewEncoder(f).Encode(spec.Windows.LayerFolders); err != nil {
-				f.Close()
-				return nil, err
-			}
-			f.Close()
-		}
-	}
-
 	// This is a Windows Argon make sure that we have a Root filled in.
 	if spec.Windows.HyperV == nil {
 		if spec.Root == nil {

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -35,6 +35,7 @@ import (
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 	"github.com/Microsoft/hcsshim/internal/hcsoci"
 	"github.com/Microsoft/hcsshim/internal/jobcontainers"
+	"github.com/Microsoft/hcsshim/internal/layers"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/internal/oc"
@@ -82,23 +83,15 @@ func newHcsStandaloneTask(ctx context.Context, events publisher, req *task.Creat
 				return nil, err
 			}
 		case *uvm.OptionsWCOW:
+			var layerFolders []string
+			if s.Windows != nil {
+				layerFolders = s.Windows.LayerFolders
+			}
 			wopts := (opts).(*uvm.OptionsWCOW)
-
-			// In order for the UVM sandbox.vhdx not to collide with the actual
-			// nested Argon sandbox.vhdx we append the \vm folder to the last
-			// entry in the list.
-			layersLen := len(s.Windows.LayerFolders)
-			layers := make([]string, layersLen)
-			copy(layers, s.Windows.LayerFolders)
-
-			vmPath := filepath.Join(layers[layersLen-1], "vm")
-			err := os.MkdirAll(vmPath, 0)
+			wopts.BootFiles, err = layers.GetWCOWUVMBootFilesFromLayers(ctx, req.Rootfs, layerFolders)
 			if err != nil {
 				return nil, err
 			}
-			layers[layersLen-1] = vmPath
-			wopts.LayerFolders = layers
-
 			parent, err = uvm.CreateWCOW(ctx, wopts)
 			if err != nil {
 				return nil, err
@@ -140,8 +133,24 @@ func createContainer(
 		resources *resources.Resources
 	)
 
+	var wcowLayers layers.WCOWLayers
+	var lcowLayers *layers.LCOWLayers
+	var layerFolders []string
+	if s.Windows != nil {
+		layerFolders = s.Windows.LayerFolders
+	}
+	if s.Linux != nil {
+		lcowLayers, err = layers.ParseLCOWLayers(rootfs, layerFolders)
+	} else {
+		wcowLayers, err = layers.ParseWCOWLayers(rootfs, layerFolders)
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+
 	if oci.IsJobContainer(s) {
-		container, resources, err = jobcontainers.Create(ctx, id, s)
+		opts := jobcontainers.CreateOptions{WCOWLayers: wcowLayers}
+		container, resources, err = jobcontainers.Create(ctx, id, s, opts)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -152,18 +161,10 @@ func createContainer(
 			Spec:             s,
 			HostingSystem:    parent,
 			NetworkNamespace: netNS,
+			LCOWLayers:       lcowLayers,
+			WCOWLayers:       wcowLayers,
 		}
-		if s.Linux != nil {
-			var layerFolders []string
-			if s.Windows != nil {
-				layerFolders = s.Windows.LayerFolders
-			}
-			lcowLayers, err := getLCOWLayers(rootfs, layerFolders)
-			if err != nil {
-				return nil, nil, err
-			}
-			opts.LCOWLayers = lcowLayers
-		}
+
 		if shimOpts != nil {
 			opts.ScaleCPULimitsToSandbox = shimOpts.ScaleCpuLimitsToSandbox
 		}