From 60677729678220d068e10bad57e2aac8a377bab2 Mon Sep 17 00:00:00 2001 From: Maksim An Date: Tue, 19 Oct 2021 01:57:16 -0700 Subject: [PATCH] Export hcsshim annotations into its own package Previously hcsshim annotations were not exported, which lead to use of hardcoded strings in various places (e.g. tests, guest code etc). This change creates a package just for the hcsshim annotations. Signed-off-by: Maksim An --- cmd/containerd-shim-runhcs-v1/pod.go | 11 +- cmd/containerd-shim-runhcs-v1/start.go | 5 +- cmd/containerd-shim-runhcs-v1/task_hcs.go | 3 +- cmd/runhcs/container.go | 5 +- internal/guest/runtime/hcsv2/nvidia_utils.go | 6 +- .../guest/runtime/hcsv2/sandbox_container.go | 3 +- internal/guest/runtime/hcsv2/spec.go | 7 +- internal/guest/runtime/hcsv2/uvm.go | 5 +- .../guest/runtime/hcsv2/workload_container.go | 3 +- internal/hcsoci/create.go | 7 +- internal/hcsoci/devices.go | 9 +- internal/hcsoci/hcsdoc_wcow.go | 19 +- internal/hcsoci/resources_lcow.go | 4 +- internal/jobcontainers/oci.go | 14 +- internal/oci/annotations.go | 237 ---------------- internal/oci/sandbox.go | 26 +- internal/oci/sandbox_test.go | 20 +- internal/oci/util.go | 7 +- internal/oci/uvm.go | 115 ++++---- internal/oci/uvm_test.go | 13 +- pkg/annotations/annotations.go | 253 ++++++++++++++++++ test/containerd-shim-runhcs-v1/start_test.go | 15 +- test/cri-containerd/clone_test.go | 12 +- .../container_layers_packing_test.go | 14 +- test/cri-containerd/container_test.go | 22 +- test/cri-containerd/container_update_test.go | 4 +- .../container_virtual_device_test.go | 28 +- test/cri-containerd/createcontainer_test.go | 18 +- test/cri-containerd/jobcontainer_test.go | 12 +- test/cri-containerd/pod_update_test.go | 8 +- test/cri-containerd/runpodsandbox_test.go | 160 +++++------ .../scale_cpu_limits_to_sandbox_test.go | 4 +- test/cri-containerd/stats_test.go | 8 +- .../hcsshim/internal/hcsoci/create.go | 7 +- .../hcsshim/internal/hcsoci/devices.go | 9 +- .../hcsshim/internal/hcsoci/hcsdoc_wcow.go | 19 +- .../hcsshim/internal/hcsoci/resources_lcow.go | 4 +- .../hcsshim/internal/oci/annotations.go | 237 ---------------- .../Microsoft/hcsshim/internal/oci/sandbox.go | 26 +- .../Microsoft/hcsshim/internal/oci/util.go | 7 +- .../Microsoft/hcsshim/internal/oci/uvm.go | 115 ++++---- .../hcsshim/pkg/annotations/annotations.go | 253 ++++++++++++++++++ test/vendor/modules.txt | 1 + 43 files changed, 901 insertions(+), 854 deletions(-) delete mode 100644 internal/oci/annotations.go create mode 100644 pkg/annotations/annotations.go delete mode 100644 test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go create mode 100644 test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index abd9cbc506..6ed3c35b8b 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" eventstypes "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/runtime" @@ -67,7 +68,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation: '%s': '%s' got '%s'", - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesContainerType, oci.KubernetesContainerTypeSandbox, ct) } @@ -75,7 +76,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation '%s': '%s' got '%s'", - oci.KubernetesSandboxIDAnnotation, + annotations.KubernetesSandboxID, req.ID, sid) } @@ -175,7 +176,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques p.host = parent if parent != nil { cid := req.ID - if id, ok := s.Annotations[oci.AnnotationNcproxyContainerID]; ok { + if id, ok := s.Annotations[annotations.NcproxyContainerID]; ok { cid = id } caAddr := fmt.Sprintf(uvm.ComputeAgentAddrFmt, cid) @@ -319,7 +320,7 @@ func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *sp return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation: '%s': '%s' got '%s'", - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesContainerType, oci.KubernetesContainerTypeContainer, ct) } @@ -327,7 +328,7 @@ func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *sp return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation '%s': '%s' got '%s'", - oci.KubernetesSandboxIDAnnotation, + annotations.KubernetesSandboxID, p.id, sid) } diff --git a/cmd/containerd-shim-runhcs-v1/start.go b/cmd/containerd-shim-runhcs-v1/start.go index e5fa54929a..520ddcb7f3 100644 --- a/cmd/containerd-shim-runhcs-v1/start.go +++ b/cmd/containerd-shim-runhcs-v1/start.go @@ -12,6 +12,7 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/containerd/containerd/runtime/v2/shim" "github.com/containerd/containerd/runtime/v2/task" "github.com/containerd/ttrpc" @@ -102,8 +103,8 @@ The start command can either start a new shim or return an address to an existin if isSandbox && idFlag != sbid { return errors.Errorf( "'id' and '%s' must match for '%s=%s'", - oci.KubernetesSandboxIDAnnotation, - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesSandboxID, + annotations.KubernetesContainerType, oci.KubernetesContainerTypeSandbox) } diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs.go b/cmd/containerd-shim-runhcs-v1/task_hcs.go index 9f6f5a9a11..4ecb35716f 100644 --- a/cmd/containerd-shim-runhcs-v1/task_hcs.go +++ b/cmd/containerd-shim-runhcs-v1/task_hcs.go @@ -37,6 +37,7 @@ import ( "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" ) const bytesPerMB = 1024 * 1024 @@ -52,7 +53,7 @@ func newHcsStandaloneTask(ctx context.Context, events publisher, req *task.Creat return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "cannot create standalone task, expected no annotation: '%s': got '%s'", - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesContainerType, ct) } diff --git a/cmd/runhcs/container.go b/cmd/runhcs/container.go index 66b0bb9589..6726606839 100644 --- a/cmd/runhcs/container.go +++ b/cmd/runhcs/container.go @@ -23,6 +23,7 @@ import ( "github.com/Microsoft/hcsshim/internal/runhcs" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "golang.org/x/sys/windows" @@ -207,8 +208,8 @@ func launchShim(cmd, pidFile, logFile string, args []string, data interface{}) ( // returns `("", false)`. func parseSandboxAnnotations(a map[string]string) (string, bool) { var t, id string - if t = a["io.kubernetes.cri.container-type"]; t != "" { - id = a["io.kubernetes.cri.sandbox-id"] + if t = a[annotations.KubernetesContainerType]; t != "" { + id = a[annotations.KubernetesSandboxID] } else if t = a["io.kubernetes.cri-o.ContainerType"]; t != "" { id = a["io.kubernetes.cri-o.SandboxID"] } else if t = a["io.kubernetes.docker.type"]; t != "" { diff --git a/internal/guest/runtime/hcsv2/nvidia_utils.go b/internal/guest/runtime/hcsv2/nvidia_utils.go index 09931368d2..206eb4fade 100644 --- a/internal/guest/runtime/hcsv2/nvidia_utils.go +++ b/internal/guest/runtime/hcsv2/nvidia_utils.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/cmd/gcstools/generichook" "github.com/Microsoft/hcsshim/internal/guest/storage/pci" + "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -19,9 +20,6 @@ import ( // this MUST match the path mapped to in the shim const lcowNvidiaMountPath = "/run/nvidia" -// annotation to find the gpu capabilities on the container spec -// must match the hcsshim annotation string for gpu capabilities -const annotationContainerGPUCapabilities = "io.microsoft.container.gpu.capabilities" const nvidiaDebugFilePath = "/nvidia-container.log" const nvidiaToolBinary = "nvidia-container-cli" @@ -50,7 +48,7 @@ func addNvidiaDevicePreHook(ctx context.Context, spec *oci.Spec) error { "configure", "--ldconfig=@/sbin/ldconfig", } - if capabilities, ok := spec.Annotations[annotationContainerGPUCapabilities]; ok { + if capabilities, ok := spec.Annotations[annotations.ContainerGPUCapabilities]; ok { caps := strings.Split(capabilities, ",") for _, c := range caps { args = append(args, fmt.Sprintf("--%s", c)) diff --git a/internal/guest/runtime/hcsv2/sandbox_container.go b/internal/guest/runtime/hcsv2/sandbox_container.go index a4bd594505..14e3366003 100644 --- a/internal/guest/runtime/hcsv2/sandbox_container.go +++ b/internal/guest/runtime/hcsv2/sandbox_container.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/guest/network" "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" @@ -112,7 +113,7 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) ( } } - if rlimCore := spec.Annotations["io.microsoft.lcow.rlimitcore"]; rlimCore != "" { + if rlimCore := spec.Annotations[annotations.RLimitCore]; rlimCore != "" { if err := setCoreRLimit(spec, rlimCore); err != nil { return err } diff --git a/internal/guest/runtime/hcsv2/spec.go b/internal/guest/runtime/hcsv2/spec.go index 99373ba75b..d53fa576ba 100644 --- a/internal/guest/runtime/hcsv2/spec.go +++ b/internal/guest/runtime/hcsv2/spec.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/user" oci "github.com/opencontainers/runtime-spec/specs-go" @@ -197,7 +198,7 @@ func getGroup(spec *oci.Spec, filter func(user.Group) bool) (user.Group, error) // applyAnnotationsToSpec modifies the spec based on additional information from annotations func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error { // Check if we need to override container's /dev/shm - if val, ok := spec.Annotations["io.microsoft.container.storage.shm.size-kb"]; ok { + if val, ok := spec.Annotations[annotations.LCOWDevShmSizeInKb]; ok { sz, err := strconv.ParseInt(val, 10, 64) if err != nil { return errors.Wrap(err, "/dev/shm size must be a valid integer") @@ -220,8 +221,8 @@ func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error { } // Check if we need to do any capability/device mappings - if spec.Annotations["io.microsoft.virtualmachine.lcow.privileged"] == "true" { - log.G(ctx).Debug("'io.microsoft.virtualmachine.lcow.privileged' set for privileged container") + if spec.Annotations[annotations.LCOWPrivileged] == "true" { + log.G(ctx).Debugf("'%s' set for privileged container", annotations.LCOWPrivileged) // Add all host devices hostDevices, err := devices.HostDevices() diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 634c94678f..d58d986057 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -26,6 +26,7 @@ import ( "github.com/Microsoft/hcsshim/internal/guest/storage/pmem" "github.com/Microsoft/hcsshim/internal/guest/storage/scsi" "github.com/Microsoft/hcsshim/internal/guest/transport" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/pkg/securitypolicy" shellwords "github.com/mattn/go-shellwords" "github.com/pkg/errors" @@ -156,7 +157,7 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM } var namespaceID string - criType, isCRI := settings.OCISpecification.Annotations["io.kubernetes.cri.container-type"] + criType, isCRI := settings.OCISpecification.Annotations[annotations.KubernetesContainerType] if isCRI { switch criType { case "sandbox": @@ -180,7 +181,7 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM return nil, err } case "container": - sid, ok := settings.OCISpecification.Annotations["io.kubernetes.cri.sandbox-id"] + sid, ok := settings.OCISpecification.Annotations[annotations.KubernetesSandboxID] if !ok || sid == "" { return nil, errors.Errorf("unsupported 'io.kubernetes.cri.sandbox-id': '%s'", sid) } diff --git a/internal/guest/runtime/hcsv2/workload_container.go b/internal/guest/runtime/hcsv2/workload_container.go index e85311fc59..913552fc3d 100644 --- a/internal/guest/runtime/hcsv2/workload_container.go +++ b/internal/guest/runtime/hcsv2/workload_container.go @@ -10,6 +10,7 @@ import ( "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runc/libcontainer/devices" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -161,7 +162,7 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci. return err } - if rlimCore := spec.Annotations["io.microsoft.lcow.rlimitcore"]; rlimCore != "" { + if rlimCore := spec.Annotations[annotations.RLimitCore]; rlimCore != "" { if err := setCoreRLimit(spec, rlimCore); err != nil { return err } diff --git a/internal/hcsoci/create.go b/internal/hcsoci/create.go index cd9ca03f7e..ae1dbdeab2 100644 --- a/internal/hcsoci/create.go +++ b/internal/hcsoci/create.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -97,13 +98,13 @@ func verifyCloneContainerSpecs(templateSpec, cloneSpec *specs.Spec) error { } // for annotations check that the values of memory & cpu annotations are same - if templateSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] != cloneSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] { + if templateSpec.Annotations[annotations.ContainerMemorySizeInMB] != cloneSpec.Annotations[annotations.ContainerMemorySizeInMB] { return fmt.Errorf("memory size limit for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorCount] != cloneSpec.Annotations[oci.AnnotationContainerProcessorCount] { + if templateSpec.Annotations[annotations.ContainerProcessorCount] != cloneSpec.Annotations[annotations.ContainerProcessorCount] { return fmt.Errorf("processor count for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorLimit] != cloneSpec.Annotations[oci.AnnotationContainerProcessorLimit] { + if templateSpec.Annotations[annotations.ContainerProcessorLimit] != cloneSpec.Annotations[annotations.ContainerProcessorLimit] { return fmt.Errorf("processor limit for template and clone containers can not be different") } diff --git a/internal/hcsoci/devices.go b/internal/hcsoci/devices.go index b007ffdd7d..cccac4397b 100644 --- a/internal/hcsoci/devices.go +++ b/internal/hcsoci/devices.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -24,8 +25,8 @@ const deviceUtilExeName = "device-util.exe" // getSpecKernelDrivers gets any device drivers specified on the spec. // Drivers are optional, therefore do not return an error if none are on the spec. -func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { - drivers := oci.ParseAnnotationCommaSeparated(oci.AnnotationVirtualMachineKernelDrivers, annotations) +func getSpecKernelDrivers(annots map[string]string) ([]string, error) { + drivers := oci.ParseAnnotationCommaSeparated(annotations.VirtualMachineKernelDrivers, annots) for _, driver := range drivers { if _, err := os.Stat(driver); err != nil { return nil, errors.Wrapf(err, "failed to find path to drivers at %s", driver) @@ -36,8 +37,8 @@ func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { // getDeviceExtensionPaths gets any device extensions paths specified on the spec. // device extensions are optional, therefore if none are on the spec, do not return an error. -func getDeviceExtensionPaths(annotations map[string]string) ([]string, error) { - extensions := oci.ParseAnnotationCommaSeparated(oci.AnnotationDeviceExtensions, annotations) +func getDeviceExtensionPaths(annots map[string]string) ([]string, error) { + extensions := oci.ParseAnnotationCommaSeparated(annotations.DeviceExtensions, annots) for _, ext := range extensions { if _, err := os.Stat(ext); err != nil { return nil, errors.Wrapf(err, "failed to find path to driver extensions at %s", ext) diff --git a/internal/hcsoci/hcsdoc_wcow.go b/internal/hcsoci/hcsdoc_wcow.go index 54c4be1eaf..8f51420540 100644 --- a/internal/hcsoci/hcsdoc_wcow.go +++ b/internal/hcsoci/hcsdoc_wcow.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/uvmfolder" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -113,17 +114,17 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount // exclusive. func ConvertCPULimits(ctx context.Context, cid string, spec *specs.Spec, maxCPUCount int32) (int32, int32, int32, error) { cpuNumSet := 0 - cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, oci.AnnotationContainerProcessorCount, 0) + cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, annotations.ContainerProcessorCount, 0) if cpuCount > 0 { cpuNumSet++ } - cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, oci.AnnotationContainerProcessorLimit, 0) + cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, annotations.ContainerProcessorLimit, 0) if cpuLimit > 0 { cpuNumSet++ } - cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, oci.AnnotationContainerProcessorWeight, 0) + cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, annotations.ContainerProcessorWeight, 0) if cpuWeight > 0 { cpuNumSet++ } @@ -244,7 +245,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Memory Resources - memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, oci.AnnotationContainerMemorySizeInMB, 0) + memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, annotations.ContainerMemorySizeInMB, 0) if memoryMaxInMB > 0 { v1.MemoryMaximumInMB = int64(memoryMaxInMB) v2Container.Memory = &hcsschema.Memory{ @@ -253,8 +254,8 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Storage Resources - storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, oci.AnnotationContainerStorageQoSBandwidthMaximum, 0) - storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, oci.AnnotationContainerStorageQoSIopsMaximum, 0) + storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, annotations.ContainerStorageQoSBandwidthMaximum, 0) + storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, annotations.ContainerStorageQoSIopsMaximum, 0) if storageBandwidthMax > 0 || storageIopsMax > 0 { v1.StorageBandwidthMaximum = uint64(storageBandwidthMax) v1.StorageIOPSMaximum = uint64(storageIopsMax) @@ -392,7 +393,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter dumpPath = coi.HostingSystem.ProcessDumpLocation() } - if specDumpPath, ok := coi.Spec.Annotations[oci.AnnotationContainerProcessDumpLocation]; ok { + if specDumpPath, ok := coi.Spec.Annotations[annotations.ContainerProcessDumpLocation]; ok { // If a process dump path was specified at pod creation time for a hypervisor isolated pod, then // use this value. If one was specified on the container creation document then override with this // instead. Unlike Linux, Windows containers can set the dump path on a per container basis. @@ -466,8 +467,8 @@ func parseAssignedDevices(ctx context.Context, coi *createOptionsInternal, v2 *h // corresponding value the registry expects to be set. // // See DumpType at https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps for the mappings -func parseDumpType(annotations map[string]string) (int32, error) { - dmpTypeStr := annotations[oci.AnnotationWCOWProcessDumpType] +func parseDumpType(annots map[string]string) (int32, error) { + dmpTypeStr := annots[annotations.WCOWProcessDumpType] switch dmpTypeStr { case "": // If no type specified, default to full dumps. diff --git a/internal/hcsoci/resources_lcow.go b/internal/hcsoci/resources_lcow.go index 1bdf795ae1..c4fec89f27 100644 --- a/internal/hcsoci/resources_lcow.go +++ b/internal/hcsoci/resources_lcow.go @@ -14,9 +14,9 @@ import ( "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -25,7 +25,7 @@ import ( // shim option is set. Right now we only support Nvidia gpus, so this will default to // a gpu vhd with nvidia files func getGPUVHDPath(coi *createOptionsInternal) (string, error) { - gpuVHDPath, ok := coi.Spec.Annotations[oci.AnnotationGPUVHDPath] + gpuVHDPath, ok := coi.Spec.Annotations[annotations.GPUVHDPath] if !ok || gpuVHDPath == "" { return "", fmt.Errorf("no gpu vhd specified %s", gpuVHDPath) } diff --git a/internal/jobcontainers/oci.go b/internal/jobcontainers/oci.go index 95dfe68a40..adb5815ca3 100644 --- a/internal/jobcontainers/oci.go +++ b/internal/jobcontainers/oci.go @@ -4,9 +4,9 @@ import ( "context" "github.com/Microsoft/hcsshim/internal/hcsoci" - "github.com/Microsoft/hcsshim/internal/processorinfo" - "github.com/Microsoft/hcsshim/internal/jobobject" + "github.com/Microsoft/hcsshim/internal/processorinfo" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/internal/oci" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -16,8 +16,8 @@ const processorWeightMax = 10000 // This file contains helpers for converting parts of the oci spec to useful // structures/limits to be applied to a job object. -func getUserTokenInheritAnnotation(annotations map[string]string) bool { - val, ok := annotations[oci.AnnotationHostProcessInheritUser] +func getUserTokenInheritAnnotation(annots map[string]string) bool { + val, ok := annots[annotations.HostProcessInheritUser] return ok && val == "true" } @@ -43,11 +43,11 @@ func specToLimits(ctx context.Context, cid string, s *specs.Spec) (*jobobject.Jo } // Memory limit - memLimitMB := oci.ParseAnnotationsMemory(ctx, s, oci.AnnotationContainerMemorySizeInMB, 0) + memLimitMB := oci.ParseAnnotationsMemory(ctx, s, annotations.ContainerMemorySizeInMB, 0) // IO limits - maxBandwidth := int64(oci.ParseAnnotationsStorageBps(ctx, s, oci.AnnotationContainerStorageQoSBandwidthMaximum, 0)) - maxIops := int64(oci.ParseAnnotationsStorageIops(ctx, s, oci.AnnotationContainerStorageQoSIopsMaximum, 0)) + maxBandwidth := int64(oci.ParseAnnotationsStorageBps(ctx, s, annotations.ContainerStorageQoSBandwidthMaximum, 0)) + maxIops := int64(oci.ParseAnnotationsStorageIops(ctx, s, annotations.ContainerStorageQoSIopsMaximum, 0)) return &jobobject.JobLimits{ CPULimit: realCPULimit, diff --git a/internal/oci/annotations.go b/internal/oci/annotations.go deleted file mode 100644 index bed5230470..0000000000 --- a/internal/oci/annotations.go +++ /dev/null @@ -1,237 +0,0 @@ -package oci - -const ( - // AnnotationContainerMemorySizeInMB overrides the container memory size set - // via the OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Memory.Limit`. - AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" - - // AnnotationContainerProcessorCount overrides the container processor count - // set via the OCI spec. - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. - AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" - - // AnnotationContainerProcessorLimit overrides the container processor limit - // set via the OCI spec. - // - // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.CPU.Maximum`. - AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" - - // AnnotationContainerProcessorWeight overrides the container processor - // weight set via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. - AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" - - // AnnotationContainerStorageQoSBandwidthMaximum overrides the container - // storage bandwidth per second set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Bps`. - AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" - - // AnnotationContainerStorageQoSIopsMaximum overrides the container storage - // maximum iops set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Iops`. - AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" - - // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd - AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" - - // AnnotationVirtualMachineKernelDrivers indicates what drivers to install in the pod. - // This value should contain a list of comma separated directories containing all - // files and information needed to install given driver(s). For windows, this may - // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. - // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. - AnnotationVirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" - - // AnnotationDeviceExtensions contains a comma separated list of full paths to device extension files. - // The content of these are added to a container's hcs create document. - AnnotationDeviceExtensions = "io.microsoft.container.wcow.deviceextensions" - - // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process - // container as and instead inherit the user token from the executable that is launching the container process. - AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" - - // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). - AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" - - // AnnotationAllowOvercommit indicates if we should allow over commit memory for UVM. - // Defaults to true. For physical backed memory, set to false. - AnnotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" - - // AnnotationEnableDeferredCommit indicates if we should allow deferred memory commit for UVM. - // Defaults to false. For virtual memory with deferred commit, set to true. - AnnotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" - - // AnnotationEnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM - // to trim non-zeroed pages from the working set (if supported by the guest operating system). - AnnotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" - - // AnnotationMemorySizeInMB overrides the container memory size set via the - // OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - AnnotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" - - // AnnotationMemoryLowMMIOGapInMB indicates the low MMIO gap in MB - AnnotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO base in MB - AnnotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO gap in MB - AnnotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" - - // annotationProcessorCount overrides the hypervisor isolated vCPU count set - // via the OCI spec. - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" - - // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set - // via the OCI spec. - // - // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" - - // AnnotationProcessorWeight overrides the hypervisor isolated vCPU weight set - // via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" - - // AnnotationVPMemCount indicates the max number of vpmem devices that can be used on the UVM - AnnotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" - - // AnnotationVPMemSize indicates the size of the VPMem devices. - AnnotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" - - // AnnotationPreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. - // valid values are "initrd" or "vhd" - AnnotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" - - // AnnotationBootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM - AnnotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" - - // AnnotationKernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` - AnnotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" - - // AnnotationVPCIEnabled indicates that pci support should be enabled for the LCOW UVM - AnnotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" - - // AnnotationVPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping - AnnotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" - - // AnnotationKernelBootOptions is used to specify kernel options used while booting a linux kernel - AnnotationKernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" - - // AnnotationStorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` - // will default to the platform default. - AnnotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" - - // AnnotationStorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will - // default to the platform default. - AnnotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" - - // AnnotationFullyPhysicallyBacked indicates that the UVM should use physically backed memory only, - // including for additional devices added later. - AnnotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" - - // AnnotationDisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM - // for WCOW. - AnnotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" - - // AnnotationVSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM - AnnotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" - - // AnnotationCPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any - AnnotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" - - // AnnotationSaveAsTemplate annotation must be used with a pod & container creation request. - // If this annotation is present in the request then it will save the UVM (pod) - // and the container(s) inside it as a template. However, this also means that this - // pod and the containers inside this pod will permananetly stay in the - // paused/templated state and can not be resumed again. - AnnotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" - - // AnnotationTemplateID should be used when creating a pod or a container from a template. - // When creating a pod from a template use the ID of the templated pod as the - // TemplateID and when creating a container use the ID of the templated container as - // the TemplateID. It is the client's responsibility to make sure that the sandbox - // within which a cloned container needs to be created must also be created from the - // same template. - AnnotationTemplateID = "io.microsoft.virtualmachine.templateid" - - // AnnotationNetworkConfigProxy holds the address of the network config proxy service. - // If set, network setup will be attempted via ncproxy. - AnnotationNetworkConfigProxy = "io.microsoft.network.ncproxy" - - // AnnotationNcproxyContainerID indicates whether or not to use the hcsshim container ID - // when setting up ncproxy and computeagent - AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" - - // AnnotationEncryptedScratchDisk indicates whether or not the container scratch disks - // should be encrypted or not - AnnotationEncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" - - // AnnotationSecurityPolicy is used to specify a security policy for opengcs to enforce - AnnotationSecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" - // AnnotationContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As - // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of - // some kind (vhd, bind mount, fileshare mount etc.) - AnnotationContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" - - // AnnotationWCOWProcessDumpType specifies the type of dump to create when generating a local user mode - // process dump for Windows containers. The supported options are "mini", and "full". - // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps - AnnotationWCOWProcessDumpType = "io.microsoft.wcow.processdumptype" - - // AnnotationRLimitCore specifies the core rlimit value for a container. This will need to be set - // in order to have core dumps generated for a given container. - AnnotationRLimitCore = "io.microsoft.lcow.rlimitcore" -) diff --git a/internal/oci/sandbox.go b/internal/oci/sandbox.go index 3c3f89280d..569b035654 100644 --- a/internal/oci/sandbox.go +++ b/internal/oci/sandbox.go @@ -2,28 +2,22 @@ package oci import ( "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" ) -// KubernetesContainerTypeAnnotation is the annotation used by CRI to define the `ContainerType`. -const KubernetesContainerTypeAnnotation = "io.kubernetes.cri.container-type" - -// KubernetesSandboxIDAnnotation is the annotation used by CRI to define the -// KubernetesContainerTypeAnnotation == "sandbox"` ID. -const KubernetesSandboxIDAnnotation = "io.kubernetes.cri.sandbox-id" - // KubernetesContainerType defines the valid types of the -// `KubernetesContainerTypeAnnotation` annotation. +// `annotations.KubernetesContainerType` annotation. type KubernetesContainerType string const ( // KubernetesContainerTypeNone is only valid when - // `KubernetesContainerTypeAnnotation` is not set. + // `annotations.KubernetesContainerType` is not set. KubernetesContainerTypeNone KubernetesContainerType = "" // KubernetesContainerTypeContainer is valid when - // `KubernetesContainerTypeAnnotation == "container"`. + // `annotations.KubernetesContainerType == "container"`. KubernetesContainerTypeContainer KubernetesContainerType = "container" // KubernetesContainerTypeSandbox is valid when - // `KubernetesContainerTypeAnnotation == "sandbox"`. + // `annotations.KubernetesContainerType == "sandbox"`. KubernetesContainerTypeSandbox KubernetesContainerType = "sandbox" ) @@ -32,27 +26,27 @@ const ( // annotations and if found validates the set before returning. func GetSandboxTypeAndID(specAnnotations map[string]string) (KubernetesContainerType, string, error) { var ct KubernetesContainerType - if t, ok := specAnnotations[KubernetesContainerTypeAnnotation]; ok { + if t, ok := specAnnotations[annotations.KubernetesContainerType]; ok { switch t { case string(KubernetesContainerTypeContainer): ct = KubernetesContainerTypeContainer case string(KubernetesContainerTypeSandbox): ct = KubernetesContainerTypeSandbox default: - return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", KubernetesContainerTypeAnnotation, t) + return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", annotations.KubernetesContainerType, t) } } - id := specAnnotations[KubernetesSandboxIDAnnotation] + id := specAnnotations[annotations.KubernetesSandboxID] switch ct { case KubernetesContainerTypeContainer, KubernetesContainerTypeSandbox: if id == "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesContainerTypeAnnotation, KubernetesSandboxIDAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesContainerType, annotations.KubernetesSandboxID) } default: if id != "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesSandboxIDAnnotation, KubernetesContainerTypeAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesSandboxID, annotations.KubernetesContainerType) } } return ct, id, nil diff --git a/internal/oci/sandbox_test.go b/internal/oci/sandbox_test.go index b88765d84e..c6d1ebb8f6 100644 --- a/internal/oci/sandbox_test.go +++ b/internal/oci/sandbox_test.go @@ -1,10 +1,14 @@ package oci -import "testing" +import ( + "testing" + + "github.com/Microsoft/hcsshim/pkg/annotations" +) func Test_GetSandboxTypeAndID_TypeContainer_NoID_Failure(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "container", + annotations.KubernetesContainerType: "container", } ct, id, err := GetSandboxTypeAndID(a) if err == nil { @@ -20,7 +24,7 @@ func Test_GetSandboxTypeAndID_TypeContainer_NoID_Failure(t *testing.T) { func Test_GetSandboxTypeAndID_TypeSandbox_NoID_Failure(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "sandbox", + annotations.KubernetesContainerType: "sandbox", } ct, id, err := GetSandboxTypeAndID(a) if err == nil { @@ -36,7 +40,7 @@ func Test_GetSandboxTypeAndID_TypeSandbox_NoID_Failure(t *testing.T) { func Test_GetSandboxTypeAndID_NoType_ValidID_Failure(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.sandbox-id": t.Name(), + annotations.KubernetesSandboxID: t.Name(), } ct, id, err := GetSandboxTypeAndID(a) if err == nil { @@ -65,8 +69,8 @@ func Test_GetSandboxTypeAndID_NoAnnotations_Success(t *testing.T) { func Test_GetSandboxTypeAndID_TypeContainer_ValidID_Success(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "container", - "io.kubernetes.cri.sandbox-id": t.Name(), + annotations.KubernetesContainerType: "container", + annotations.KubernetesSandboxID: t.Name(), } ct, id, err := GetSandboxTypeAndID(a) if err != nil { @@ -82,8 +86,8 @@ func Test_GetSandboxTypeAndID_TypeContainer_ValidID_Success(t *testing.T) { func Test_GetSandboxTypeAndID_TypeSandbox_ValidID_Success(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "sandbox", - "io.kubernetes.cri.sandbox-id": t.Name(), + annotations.KubernetesContainerType: "sandbox", + annotations.KubernetesSandboxID: t.Name(), } ct, id, err := GetSandboxTypeAndID(a) if err != nil { diff --git a/internal/oci/util.go b/internal/oci/util.go index 85f52016ff..6452241eca 100644 --- a/internal/oci/util.go +++ b/internal/oci/util.go @@ -1,6 +1,9 @@ package oci -import "github.com/opencontainers/runtime-spec/specs-go" +import ( + "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/opencontainers/runtime-spec/specs-go" +) // IsLCOW checks if `s` is a LCOW config. func IsLCOW(s *specs.Spec) bool { @@ -19,5 +22,5 @@ func IsIsolated(s *specs.Spec) bool { // IsJobContainer checks if `s` is asking for a Windows job container. func IsJobContainer(s *specs.Spec) bool { - return s.Annotations[AnnotationHostProcessContainer] == "true" + return s.Annotations[annotations.HostProcessContainer] == "true" } diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index c77a90731a..fcf41fba08 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "strings" @@ -219,13 +220,13 @@ func parseAnnotationsString(a map[string]string, key string, def string) string // if this create request should be considered as a template creation request. If value // is found the returns the actual value, returns false otherwise. func ParseAnnotationsSaveAsTemplate(ctx context.Context, s *specs.Spec) bool { - return parseAnnotationsBool(ctx, s.Annotations, AnnotationSaveAsTemplate, false) + return parseAnnotationsBool(ctx, s.Annotations, annotations.SaveAsTemplate, false) } // ParseAnnotationsTemplateID searches for the templateID in the create request. If the // value is found then returns the value otherwise returns the empty string. func ParseAnnotationsTemplateID(ctx context.Context, s *specs.Spec) string { - return parseAnnotationsString(s.Annotations, AnnotationTemplateID, "") + return parseAnnotationsString(s.Annotations, annotations.TemplateID, "") } func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, templateID string, err error) { @@ -244,7 +245,7 @@ func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, // handleAnnotationKernelDirectBoot handles parsing annotationKernelDirectBoot and setting // implied annotations from the result. func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.KernelDirect = parseAnnotationsBool(ctx, a, AnnotationKernelDirectBoot, lopts.KernelDirect) + lopts.KernelDirect = parseAnnotationsBool(ctx, a, annotations.KernelDirectBoot, lopts.KernelDirect) if !lopts.KernelDirect { lopts.KernelFile = uvm.KernelFile } @@ -253,7 +254,7 @@ func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, // handleAnnotationPreferredRootFSType handles parsing annotationPreferredRootFSType and setting // implied annotations from the result func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, AnnotationPreferredRootFSType, lopts.PreferredRootFSType) + lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, annotations.PreferredRootFSType, lopts.PreferredRootFSType) switch lopts.PreferredRootFSType { case uvm.PreferredRootFSTypeInitRd: lopts.RootFSFile = uvm.InitrdFile @@ -267,7 +268,7 @@ func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]strin func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]string, opts interface{}) { switch options := opts.(type) { case *uvm.OptionsLCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false options.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd @@ -275,7 +276,7 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str options.VPMemDeviceCount = 0 } case *uvm.OptionsWCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false } @@ -285,8 +286,8 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str // handleCloneAnnotations handles parsing annotations related to template creation and cloning // Since late cloning is only supported for WCOW this function only deals with WCOW options. func handleCloneAnnotations(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) (err error) { - wopts.IsTemplate = parseAnnotationsBool(ctx, a, AnnotationSaveAsTemplate, false) - templateID := parseAnnotationsString(a, AnnotationTemplateID, "") + wopts.IsTemplate = parseAnnotationsBool(ctx, a, annotations.SaveAsTemplate, false) + templateID := parseAnnotationsString(a, annotations.TemplateID, "") if templateID != "" { tc, err := clone.FetchTemplateConfig(ctx, templateID) if err != nil { @@ -310,29 +311,29 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( } if IsLCOW(s) { lopts := uvm.NewDefaultOptionsLCOW(id, owner) - lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, lopts.MemorySizeInMB) - lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) - lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) - lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) - lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, lopts.AllowOvercommit) - lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, lopts.EnableDeferredCommit) - lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) - lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, lopts.ProcessorCount) - lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, lopts.ProcessorLimit) - lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, lopts.ProcessorWeight) - lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, AnnotationVPMemCount, lopts.VPMemDeviceCount) - lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, AnnotationVPMemSize, lopts.VPMemSizeBytes) - lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) - lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) - lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) - lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPCIEnabled, lopts.VPCIEnabled) - lopts.BootFilesPath = parseAnnotationsString(s.Annotations, AnnotationBootFilesRootPath, lopts.BootFilesPath) - lopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, lopts.CPUGroupID) - lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, lopts.NetworkConfigProxy) - lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, AnnotationEncryptedScratchDisk, lopts.EnableScratchEncryption) - lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, AnnotationSecurityPolicy, lopts.SecurityPolicy) - lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, AnnotationKernelBootOptions, lopts.KernelBootOptions) - lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, lopts.ProcessDumpLocation) + lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, lopts.MemorySizeInMB) + lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) + lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) + lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) + lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, lopts.AllowOvercommit) + lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, lopts.EnableDeferredCommit) + lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableColdDiscardHint, lopts.EnableColdDiscardHint) + lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, lopts.ProcessorCount) + lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, lopts.ProcessorLimit) + lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, lopts.ProcessorWeight) + lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, annotations.VPMemCount, lopts.VPMemDeviceCount) + lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, annotations.VPMemSize, lopts.VPMemSizeBytes) + lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, annotations.VPMemNoMultiMapping, lopts.VPMemNoMultiMapping) + lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) + lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) + lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotations.VPCIEnabled, lopts.VPCIEnabled) + lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotations.BootFilesRootPath, lopts.BootFilesPath) + lopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, lopts.CPUGroupID) + lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, lopts.NetworkConfigProxy) + lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, annotations.EncryptedScratchDisk, lopts.EnableScratchEncryption) + lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, annotations.SecurityPolicy, lopts.SecurityPolicy) + lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) + lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, lopts.ProcessDumpLocation) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) @@ -342,22 +343,22 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( return lopts, nil } else if IsWCOW(s) { wopts := uvm.NewDefaultOptionsWCOW(id, owner) - wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, wopts.MemorySizeInMB) - wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) - wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) - wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) - wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, wopts.AllowOvercommit) - wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, wopts.EnableDeferredCommit) - wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, wopts.ProcessorCount) - wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, wopts.ProcessorLimit) - wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, wopts.ProcessorWeight) - wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) - wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) - wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, AnnotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) - wopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, wopts.CPUGroupID) - wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, wopts.NetworkConfigProxy) - wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, AnnotationVSMBNoDirectMap, wopts.NoDirectMap) - wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, wopts.ProcessDumpLocation) + wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, wopts.MemorySizeInMB) + wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) + wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) + wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) + wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, wopts.AllowOvercommit) + wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, wopts.EnableDeferredCommit) + wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, wopts.ProcessorCount) + wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, wopts.ProcessorLimit) + wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, wopts.ProcessorWeight) + wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) + wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) + wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, annotations.DisableCompartmentNamespace, wopts.DisableCompartmentNamespace) + wopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, wopts.CPUGroupID) + wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, wopts.NetworkConfigProxy) + wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, annotations.VSMBNoDirectMap, wopts.NoDirectMap) + wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, wopts.ProcessDumpLocation) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) if err := handleCloneAnnotations(ctx, s.Annotations, wopts); err != nil { return nil, err @@ -374,24 +375,24 @@ func UpdateSpecFromOptions(s specs.Spec, opts *runhcsopts.Options) specs.Spec { return s } - if _, ok := s.Annotations[AnnotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { - s.Annotations[AnnotationBootFilesRootPath] = opts.BootFilesRootPath + if _, ok := s.Annotations[annotations.BootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { + s.Annotations[annotations.BootFilesRootPath] = opts.BootFilesRootPath } - if _, ok := s.Annotations[AnnotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { - s.Annotations[AnnotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) + if _, ok := s.Annotations[annotations.ProcessorCount]; !ok && opts.VmProcessorCount != 0 { + s.Annotations[annotations.ProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) } - if _, ok := s.Annotations[AnnotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { - s.Annotations[AnnotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) + if _, ok := s.Annotations[annotations.MemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { + s.Annotations[annotations.MemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) } - if _, ok := s.Annotations[AnnotationGPUVHDPath]; !ok && opts.GPUVHDPath != "" { - s.Annotations[AnnotationGPUVHDPath] = opts.GPUVHDPath + if _, ok := s.Annotations[annotations.GPUVHDPath]; !ok && opts.GPUVHDPath != "" { + s.Annotations[annotations.GPUVHDPath] = opts.GPUVHDPath } - if _, ok := s.Annotations[AnnotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { - s.Annotations[AnnotationNetworkConfigProxy] = opts.NCProxyAddr + if _, ok := s.Annotations[annotations.NetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { + s.Annotations[annotations.NetworkConfigProxy] = opts.NCProxyAddr } return s diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 14b1691951..8653d9142b 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -4,6 +4,7 @@ import ( "testing" runhcsopts "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -15,12 +16,12 @@ func Test_SpecUpdate_MemorySize_WithAnnotation_WithOpts(t *testing.T) { s := &specs.Spec{ Linux: &specs.Linux{}, Annotations: map[string]string{ - AnnotationMemorySizeInMB: "2048", + annotations.MemorySizeInMB: "2048", }, } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationMemorySizeInMB] != "2048" { + if updatedSpec.Annotations[annotations.MemorySizeInMB] != "2048" { t.Fatal("should not have updated annotation to default when annotation is provided in the spec") } } @@ -36,7 +37,7 @@ func Test_SpecUpdate_MemorySize_NoAnnotation_WithOpts(t *testing.T) { } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationMemorySizeInMB] != "3072" { + if updatedSpec.Annotations[annotations.MemorySizeInMB] != "3072" { t.Fatal("should have updated annotation to default when annotation is not provided in the spec") } } @@ -49,12 +50,12 @@ func Test_SpecUpdate_ProcessorCount_WithAnnotation_WithOpts(t *testing.T) { s := &specs.Spec{ Linux: &specs.Linux{}, Annotations: map[string]string{ - AnnotationProcessorCount: "8", + annotations.ProcessorCount: "8", }, } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationProcessorCount] != "8" { + if updatedSpec.Annotations[annotations.ProcessorCount] != "8" { t.Fatal("should not have updated annotation to default when annotation is provided in the spec") } } @@ -70,7 +71,7 @@ func Test_SpecUpdate_ProcessorCount_NoAnnotation_WithOpts(t *testing.T) { } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationProcessorCount] != "4" { + if updatedSpec.Annotations[annotations.ProcessorCount] != "4" { t.Fatal("should have updated annotation to default when annotation is not provided in the spec") } } diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go new file mode 100644 index 0000000000..0042b3d24c --- /dev/null +++ b/pkg/annotations/annotations.go @@ -0,0 +1,253 @@ +package annotations + +const ( + // ContainerMemorySizeInMB overrides the container memory size set + // via the OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Memory.Limit`. + ContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" + + // ContainerProcessorCount overrides the container processor count + // set via the OCI spec. + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. + ContainerProcessorCount = "io.microsoft.container.processor.count" + + // ContainerProcessorLimit overrides the container processor limit + // set via the OCI spec. + // + // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.CPU.Maximum`. + ContainerProcessorLimit = "io.microsoft.container.processor.limit" + + // ContainerProcessorWeight overrides the container processor + // weight set via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. + ContainerProcessorWeight = "io.microsoft.container.processor.weight" + + // ContainerStorageQoSBandwidthMaximum overrides the container + // storage bandwidth per second set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Bps`. + ContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" + + // ContainerStorageQoSIopsMaximum overrides the container storage + // maximum iops set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Iops`. + ContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" + + // GPUVHDPath overrides the default path to search for the gpu vhd + GPUVHDPath = "io.microsoft.lcow.gpuvhdpath" + + // ContainerGPUCapabilities is used to find the gpu capabilities on the container spec + ContainerGPUCapabilities = "io.microsoft.container.gpu.capabilities" + + // VirtualMachineKernelDrivers indicates what drivers to install in the pod. + // This value should contain a list of comma separated directories containing all + // files and information needed to install given driver(s). For windows, this may + // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. + // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. + VirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" + + // DeviceExtensions contains a comma separated list of full paths to device extension files. + // The content of these are added to a container's hcs create document. + DeviceExtensions = "io.microsoft.container.wcow.deviceextensions" + + // HostProcessInheritUser indicates whether to ignore the username passed in to run a host process + // container as and instead inherit the user token from the executable that is launching the container process. + HostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" + + // HostProcessContainer indicates to launch a host process container (job container in this repository). + HostProcessContainer = "microsoft.com/hostprocess-container" + + // AllowOvercommit indicates if we should allow over commit memory for UVM. + // Defaults to true. For physical backed memory, set to false. + AllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" + + // EnableDeferredCommit indicates if we should allow deferred memory commit for UVM. + // Defaults to false. For virtual memory with deferred commit, set to true. + EnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" + + // EnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM + // to trim non-zeroed pages from the working set (if supported by the guest operating system). + EnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" + + // MemorySizeInMB overrides the container memory size set via the + // OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + MemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" + + // MemoryLowMMIOGapInMB indicates the low MMIO gap in MB + MemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO base in MB + MemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO gap in MB + MemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" + + // ProcessorCount overrides the hypervisor isolated vCPU count set + // via the OCI spec. + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" + + // ProcessorLimit overrides the hypervisor isolated vCPU limit set + // via the OCI spec. + // + // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" + + // ProcessorWeight overrides the hypervisor isolated vCPU weight set + // via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" + + // VPMemCount indicates the max number of vpmem devices that can be used on the UVM + VPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" + + // VPMemSize indicates the size of the VPMem devices. + VPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" + + // PreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. + // valid values are "initrd" or "vhd" + PreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" + + // BootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM + BootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" + + // KernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` + KernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" + + // VPCIEnabled indicates that pci support should be enabled for the LCOW UVM + VPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" + + // VPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping + VPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" + + // KernelBootOptions is used to specify kernel options used while booting a linux kernel + KernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" + + // StorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` + // will default to the platform default. + StorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" + + // StorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will + // default to the platform default. + StorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" + + // FullyPhysicallyBacked indicates that the UVM should use physically backed memory only, + // including for additional devices added later. + FullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" + + // DisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM + // for WCOW. + DisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" + + // VSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM + VSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" + + // CPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any + CPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" + + // SaveAsTemplate annotation must be used with a pod & container creation request. + // If this annotation is present in the request then it will save the UVM (pod) + // and the container(s) inside it as a template. However, this also means that this + // pod and the containers inside this pod will permananetly stay in the + // paused/templated state and can not be resumed again. + SaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" + + // TemplateID should be used when creating a pod or a container from a template. + // When creating a pod from a template use the ID of the templated pod as the + // TemplateID and when creating a container use the ID of the templated container as + // the TemplateID. It is the client's responsibility to make sure that the sandbox + // within which a cloned container needs to be created must also be created from the + // same template. + TemplateID = "io.microsoft.virtualmachine.templateid" + + // NetworkConfigProxy holds the address of the network config proxy service. + // If set, network setup will be attempted via ncproxy. + NetworkConfigProxy = "io.microsoft.network.ncproxy" + + // NcproxyContainerID indicates whether or not to use the hcsshim container ID + // when setting up ncproxy and computeagent + NcproxyContainerID = "io.microsoft.network.ncproxy.containerid" + + // EncryptedScratchDisk indicates whether or not the container scratch disks + // should be encrypted or not + EncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + + // SecurityPolicy is used to specify a security policy for opengcs to enforce + SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // ContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As + // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of + // some kind (vhd, bind mount, fileshare mount etc.) + ContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" + + // WCOWProcessDumpType specifies the type of dump to create when generating a local user mode + // process dump for Windows containers. The supported options are "mini", and "full". + // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps + WCOWProcessDumpType = "io.microsoft.wcow.processdumptype" + + // RLimitCore specifies the core rlimit value for a container. This will need to be set + // in order to have core dumps generated for a given container. + RLimitCore = "io.microsoft.lcow.rlimitcore" + + // LCOWDevShmSizeInKb specifies the size of LCOW /dev/shm. + LCOWDevShmSizeInKb = "io.microsoft.lcow.shm.size-kb" + + // LCOWPrivileged is used to specify that the container should be run in privileged mode + LCOWPrivileged = "io.microsoft.virtualmachine.lcow.privileged" + + // KubernetesContainerType is the annotation used by CRI to define the `ContainerType`. + KubernetesContainerType = "io.kubernetes.cri.container-type" + + // KubernetesSandboxID is the annotation used by CRI to define the + // KubernetesContainerType == "sandbox"` ID. + KubernetesSandboxID = "io.kubernetes.cri.sandbox-id" +) diff --git a/test/containerd-shim-runhcs-v1/start_test.go b/test/containerd-shim-runhcs-v1/start_test.go index 4ff5dc7cdc..91249956c4 100644 --- a/test/containerd-shim-runhcs-v1/start_test.go +++ b/test/containerd-shim-runhcs-v1/start_test.go @@ -15,6 +15,7 @@ import ( "testing" "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/containerd/containerd/runtime/v2/task" "github.com/containerd/ttrpc" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -164,8 +165,8 @@ func Test_Start_Pod_Config(t *testing.T) { t.Fatalf("failed to generate Windows config with error: %v", err) } // Setup the POD annotations - g.AddAnnotation("io.kubernetes.cri.container-type", "sandbox") - g.AddAnnotation("io.kubernetes.cri.sandbox-id", t.Name()) + g.AddAnnotation(annotations.KubernetesContainerType, "sandbox") + g.AddAnnotation(annotations.KubernetesSandboxID, t.Name()) writeBundleConfig(t, cmd.Dir, g.Config) @@ -183,8 +184,8 @@ func Test_Start_Container_InPod_Config(t *testing.T) { t.Fatalf("failed to generate Windows config with error: %v", perr) } - pg.AddAnnotation("io.kubernetes.cri.container-type", "sandbox") - pg.AddAnnotation("io.kubernetes.cri.sandbox-id", podID) + pg.AddAnnotation(annotations.KubernetesContainerType, "sandbox") + pg.AddAnnotation(annotations.KubernetesSandboxID, podID) writeBundleConfig(t, pcmd.Dir, pg.Config) @@ -203,8 +204,8 @@ func Test_Start_Container_InPod_Config(t *testing.T) { } // Setup the POD Workload container annotations - wg.AddAnnotation("io.kubernetes.cri.container-type", "container") - wg.AddAnnotation("io.kubernetes.cri.sandbox-id", podID) + wg.AddAnnotation(annotations.KubernetesContainerType, "container") + wg.AddAnnotation(annotations.KubernetesSandboxID, podID) writeBundleConfig(t, wcmd.Dir, wg.Config) @@ -222,7 +223,7 @@ func Test_Start_Container_InPod_Config_PodShim_Gone(t *testing.T) { podID := "POD-TEST" // Setup the POD Workload container annotations - g.AddAnnotation("io.kubernetes.cri.container-type", "container") + g.AddAnnotation(annotations.KubernetesContainerType, "container") g.AddAnnotation("io.kubernetes.cri.sandbox-id", podID) writeBundleConfig(t, cmd.Dir, g.Config) diff --git a/test/cri-containerd/clone_test.go b/test/cri-containerd/clone_test.go index a816e29352..01e675f846 100644 --- a/test/cri-containerd/clone_test.go +++ b/test/cri-containerd/clone_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -27,7 +27,7 @@ func getTemplatePodConfig(name string) *runtime.RunPodSandboxRequest { Namespace: testNamespace, }, Annotations: map[string]string{ - oci.AnnotationSaveAsTemplate: "true", + annotations.SaveAsTemplate: "true", }, }, RuntimeHandler: wcowHypervisorRuntimeHandler, @@ -50,7 +50,7 @@ func getTemplateContainerConfig(name string) *runtime.CreateContainerRequest { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationSaveAsTemplate: "true", + annotations.SaveAsTemplate: "true", }, }, } @@ -85,7 +85,7 @@ func getClonedPodConfig(uniqueID int, templateid string) *runtime.RunPodSandboxR Namespace: testNamespace, }, Annotations: map[string]string{ - oci.AnnotationTemplateID: templateid + "@vm", + annotations.TemplateID: templateid + "@vm", }, }, RuntimeHandler: wcowHypervisorRuntimeHandler, @@ -109,7 +109,7 @@ func getClonedContainerConfig(uniqueID int, templateid string) *runtime.CreateCo "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationTemplateID: templateid, + annotations.TemplateID: templateid, }, }, } @@ -488,7 +488,7 @@ func Test_VerifyCloneAndTemplateConfig(t *testing.T) { // change pod config to make sure the request fails cloneSandboxRequest := getClonedPodConfig(0, templatePodID) - cloneSandboxRequest.Config.Annotations[oci.AnnotationAllowOvercommit] = "false" + cloneSandboxRequest.Config.Annotations[annotations.AllowOvercommit] = "false" _, err := client.RunPodSandbox(ctx, cloneSandboxRequest) if err == nil { diff --git a/test/cri-containerd/container_layers_packing_test.go b/test/cri-containerd/container_layers_packing_test.go index 6006229fbb..63f8a1bcf8 100644 --- a/test/cri-containerd/container_layers_packing_test.go +++ b/test/cri-containerd/container_layers_packing_test.go @@ -10,9 +10,9 @@ import ( "strings" "testing" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" ) @@ -101,10 +101,10 @@ func Test_Container_Layer_Packing_On_VPMem(t *testing.T) { }, } { t.Run(fmt.Sprintf("PreferredRootFSType-%s", scenario.rootfsType), func(t *testing.T) { - annotations := map[string]string{ - oci.AnnotationPreferredRootFSType: scenario.rootfsType, + annots := map[string]string{ + annotations.PreferredRootFSType: scenario.rootfsType, } - podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, podReq) defer removePodSandbox(t, client, ctx, podID) @@ -170,10 +170,10 @@ func Test_Annotation_Disable_Multi_Mapping(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, alpine70ExtraLayers}) - annotations := map[string]string{ - oci.AnnotationVPMemNoMultiMapping: "true", + annots := map[string]string{ + annotations.VPMemNoMultiMapping: "true", } - podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, podReq) defer removePodSandbox(t, client, ctx, podID) diff --git a/test/cri-containerd/container_test.go b/test/cri-containerd/container_test.go index a603c74c41..c89f4c7cc8 100644 --- a/test/cri-containerd/container_test.go +++ b/test/cri-containerd/container_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/sirupsen/logrus" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -268,8 +268,8 @@ func Test_RunContainer_ZeroVPMEM_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "initrd", - oci.AnnotationVPMemCount: "0", + annotations.PreferredRootFSType: "initrd", + annotations.VPMemCount: "0", }), ) @@ -310,8 +310,8 @@ func Test_RunContainer_ZeroVPMEM_Multiple_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "initrd", - oci.AnnotationVPMemCount: "0", + annotations.PreferredRootFSType: "initrd", + annotations.VPMemCount: "0", }), ) @@ -794,7 +794,7 @@ func Test_CreateContainer_DevShmSize(t *testing.T) { // the /dev/shm size is expected to be in KB, set it to 256 MB size := 256 * 1024 contReq1.Config.Annotations = map[string]string{ - "io.microsoft.container.storage.shm.size-kb": strconv.Itoa(size), + annotations.LCOWDevShmSizeInKb: strconv.Itoa(size), } containerID1 := createContainer(t, client, ctx, contReq1) defer removeContainer(t, client, ctx, containerID1) @@ -836,12 +836,12 @@ func Test_CreateContainer_HugePageMount_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - annotations := map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", - oci.AnnotationMemorySizeInMB: "2048", - oci.AnnotationKernelBootOptions: "hugepagesz=2M hugepages=10", + annots := map[string]string{ + annotations.FullyPhysicallyBacked: "true", + annotations.MemorySizeInMB: "2048", + annotations.KernelBootOptions: "hugepagesz=2M hugepages=10", } - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sandboxRequest) defer removePodSandbox(t, client, ctx, podID) diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index 102e0745ec..0fa049d8e5 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -7,8 +7,8 @@ import ( "fmt" "testing" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -312,7 +312,7 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { }, Command: test.cmd, Annotations: map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), // 768MB + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), // 768MB }, }, PodSandboxId: podID, diff --git a/test/cri-containerd/container_virtual_device_test.go b/test/cri-containerd/container_virtual_device_test.go index 155e433439..8a477639a2 100644 --- a/test/cri-containerd/container_virtual_device_test.go +++ b/test/cri-containerd/container_virtual_device_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -116,16 +116,16 @@ func findTestVirtualDeviceID() (string, error) { } var lcowPodGPUAnnotations = map[string]string{ - oci.AnnotationKernelDirectBoot: "false", - oci.AnnotationAllowOvercommit: "false", - oci.AnnotationPreferredRootFSType: "initrd", - oci.AnnotationVPMemCount: "0", - oci.AnnotationVPCIEnabled: "true", + annotations.KernelDirectBoot: "false", + annotations.AllowOvercommit: "false", + annotations.PreferredRootFSType: "initrd", + annotations.VPMemCount: "0", + annotations.VPCIEnabled: "true", // we believe this is a sufficiently large high MMIO space amount for this test. // if a given gpu device needs more, this test will fail to create the container // and may hang. - oci.AnnotationMemoryHighMMIOGapInMB: "64000", - oci.AnnotationBootFilesRootPath: testGPUBootFiles, + annotations.MemoryHighMMIOGapInMB: "64000", + annotations.BootFilesRootPath: testGPUBootFiles, } func getGPUContainerRequestLCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig, device *runtime.Device) *runtime.CreateContainerRequest { @@ -145,7 +145,7 @@ func getGPUContainerRequestLCOW(t *testing.T, podID string, podConfig *runtime.P }, Linux: &runtime.LinuxContainerConfig{}, Annotations: map[string]string{ - "io.microsoft.container.gpu.capabilities": "utility", + annotations.ContainerGPUCapabilities: "utility", }, }, PodSandboxId: podID, @@ -179,7 +179,7 @@ func getGPUContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P }, }, Annotations: map[string]string{ - oci.AnnotationVirtualMachineKernelDrivers: testDriversPath, + annotations.VirtualMachineKernelDrivers: testDriversPath, }, }, PodSandboxId: podID, @@ -512,7 +512,7 @@ func Test_RunContainer_VirtualDevice_GPU_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) @@ -560,7 +560,7 @@ func Test_RunContainer_VirtualDevice_GPU_and_NoGPU_WCOW_Hypervisor(t *testing.T) t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) @@ -625,7 +625,7 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_WCOW_Hypervisor(t *testing.T) t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) @@ -680,7 +680,7 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_Removal_WCOW_Hypervisor(t *tes t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) diff --git a/test/cri-containerd/createcontainer_test.go b/test/cri-containerd/createcontainer_test.go index aaaebfb804..aaff8fcd68 100644 --- a/test/cri-containerd/createcontainer_test.go +++ b/test/cri-containerd/createcontainer_test.go @@ -11,8 +11,8 @@ import ( "testing" "github.com/Microsoft/go-winio" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -305,7 +305,7 @@ func Test_CreateContainer_MemorySize_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB }, }, } @@ -365,7 +365,7 @@ func Test_CreateContainer_MemorySize_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB }, }, } @@ -452,7 +452,7 @@ func Test_CreateContainer_CPUCount_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorCount: "1", + annotations.ContainerProcessorCount: "1", }, }, } @@ -512,7 +512,7 @@ func Test_CreateContainer_CPUCount_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorCount: "1", + annotations.ContainerProcessorCount: "1", }, }, } @@ -599,7 +599,7 @@ func Test_CreateContainer_CPULimit_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorLimit: "9000", + annotations.ContainerProcessorLimit: "9000", }, }, } @@ -659,7 +659,7 @@ func Test_CreateContainer_CPULimit_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorLimit: "9000", + annotations.ContainerProcessorLimit: "9000", }, }, } @@ -747,7 +747,7 @@ func Test_CreateContainer_CPUWeight_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorWeight: "500", + annotations.ContainerProcessorWeight: "500", }, }, } @@ -807,7 +807,7 @@ func Test_CreateContainer_CPUWeight_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorLimit: "500", + annotations.ContainerProcessorLimit: "500", }, }, } diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index 1113e4bae8..5c82248f33 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -15,7 +15,7 @@ import ( "github.com/Microsoft/go-winio/vhd" "github.com/Microsoft/hcsshim/hcn" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -24,7 +24,7 @@ func getJobContainerPodRequestWCOW(t *testing.T) *runtime.RunPodSandboxRequest { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationHostProcessContainer: "true", + annotations.HostProcessContainer: "true", }), ) } @@ -47,8 +47,8 @@ func getJobContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P }, Mounts: mounts, Annotations: map[string]string{ - oci.AnnotationHostProcessContainer: "true", - oci.AnnotationHostProcessInheritUser: "true", + annotations.HostProcessContainer: "true", + annotations.HostProcessInheritUser: "true", }, Windows: &runtime.WindowsContainerConfig{}, }, @@ -505,8 +505,8 @@ func Test_RunContainer_WorkingDirectory_JobContainer_WCOW(t *testing.T) { Command: test.cmd, WorkingDir: test.workDir, Annotations: map[string]string{ - oci.AnnotationHostProcessContainer: "true", - oci.AnnotationHostProcessInheritUser: "true", + annotations.HostProcessContainer: "true", + annotations.HostProcessInheritUser: "true", }, Windows: &runtime.WindowsContainerConfig{}, }, diff --git a/test/cri-containerd/pod_update_test.go b/test/cri-containerd/pod_update_test.go index b80efff646..a80c536c36 100644 --- a/test/cri-containerd/pod_update_test.go +++ b/test/cri-containerd/pod_update_test.go @@ -7,7 +7,7 @@ import ( "fmt" "testing" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -47,7 +47,7 @@ func Test_Pod_UpdateResources_Memory(t *testing.T) { t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), }), ) @@ -118,8 +118,8 @@ func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), + annotations.FullyPhysicallyBacked: "true", + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), }), ) diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index 83f75ea408..28261f8425 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -6,6 +6,7 @@ import ( "bufio" "context" "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" "io/ioutil" "os" "path/filepath" @@ -17,7 +18,6 @@ import ( "github.com/Microsoft/hcsshim/internal/cpugroup" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/lcow" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/processorinfo" "github.com/Microsoft/hcsshim/osversion" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" @@ -121,7 +121,7 @@ func Test_RunPodSandbox_VirtualMemory_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annotations.AllowOvercommit: "true", }), ) runPodSandboxTest(t, request) @@ -136,7 +136,7 @@ func Test_RunPodSandbox_VirtualMemory_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annotations.AllowOvercommit: "true", }), ) runPodSandboxTest(t, request) @@ -151,8 +151,8 @@ func Test_RunPodSandbox_VirtualMemory_DeferredCommit_WCOW_Hypervisor(t *testing. t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", - oci.AnnotationEnableDeferredCommit: "true", + annotations.AllowOvercommit: "true", + annotations.EnableDeferredCommit: "true", }), ) runPodSandboxTest(t, request) @@ -167,8 +167,8 @@ func Test_RunPodSandbox_VirtualMemory_DeferredCommit_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", - oci.AnnotationEnableDeferredCommit: "true", + annotations.AllowOvercommit: "true", + annotations.EnableDeferredCommit: "true", }), ) runPodSandboxTest(t, request) @@ -183,7 +183,7 @@ func Test_RunPodSandbox_PhysicalMemory_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "false", + annotations.AllowOvercommit: "false", }), ) runPodSandboxTest(t, request) @@ -198,7 +198,7 @@ func Test_RunPodSandbox_FullyPhysicallyBacked_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) runPodSandboxTest(t, request) @@ -213,7 +213,7 @@ func Test_RunPodSandbox_VSMBNoDirectMap_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationVSMBNoDirectMap: "true", + annotations.VSMBNoDirectMap: "true", }), ) runPodSandboxTest(t, request) @@ -228,7 +228,7 @@ func Test_RunPodSandbox_PhysicalMemory_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "false", + annotations.AllowOvercommit: "false", }), ) runPodSandboxTest(t, request) @@ -243,7 +243,7 @@ func Test_RunPodSandbox_FullyPhysicallyBacked_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) runPodSandboxTest(t, request) @@ -258,7 +258,7 @@ func Test_RunPodSandbox_MemorySize_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerMemorySizeInMB: "128", + annotations.ContainerMemorySizeInMB: "128", }), ) runPodSandboxTest(t, request) @@ -273,7 +273,7 @@ func Test_RunPodSandbox_MemorySize_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemorySizeInMB: "768", // 128 is too small for WCOW. It is really slow boot. + annotations.MemorySizeInMB: "768", // 128 is too small for WCOW. It is really slow boot. }), ) runPodSandboxTest(t, request) @@ -288,7 +288,7 @@ func Test_RunPodSandbox_MemorySize_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemorySizeInMB: "200", + annotations.MemorySizeInMB: "200", }), ) runPodSandboxTest(t, request) @@ -306,9 +306,9 @@ func Test_RunPodSandbox_MMIO_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemoryLowMMIOGapInMB: "100", - oci.AnnotationMemoryHighMMIOBaseInMB: "100", - oci.AnnotationMemoryHighMMIOGapInMB: "100", + annotations.MemoryLowMMIOGapInMB: "100", + annotations.MemoryHighMMIOBaseInMB: "100", + annotations.MemoryHighMMIOGapInMB: "100", }), ) runPodSandboxTest(t, request) @@ -326,9 +326,9 @@ func Test_RunPodSandbox_MMIO_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemoryLowMMIOGapInMB: "100", - oci.AnnotationMemoryHighMMIOBaseInMB: "100", - oci.AnnotationMemoryHighMMIOGapInMB: "100", + annotations.MemoryLowMMIOGapInMB: "100", + annotations.MemoryHighMMIOBaseInMB: "100", + annotations.MemoryHighMMIOGapInMB: "100", }), ) runPodSandboxTest(t, request) @@ -346,9 +346,9 @@ func Test_RunPodSandbox_MMIO_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemoryLowMMIOGapInMB: "100", - oci.AnnotationMemoryHighMMIOBaseInMB: "100", - oci.AnnotationMemoryHighMMIOGapInMB: "100", + annotations.MemoryLowMMIOGapInMB: "100", + annotations.MemoryHighMMIOBaseInMB: "100", + annotations.MemoryHighMMIOGapInMB: "100", }), ) runPodSandboxTest(t, request) @@ -363,7 +363,7 @@ func Test_RunPodSandbox_CPUCount_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorCount: "1", + annotations.ContainerProcessorCount: "1", }), ) runPodSandboxTest(t, request) @@ -378,7 +378,7 @@ func Test_RunPodSandbox_CPUCount_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorCount: "1", + annotations.ProcessorCount: "1", }), ) runPodSandboxTest(t, request) @@ -393,7 +393,7 @@ func Test_RunPodSandbox_CPUCount_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorCount: "1", + annotations.ProcessorCount: "1", }), ) runPodSandboxTest(t, request) @@ -408,7 +408,7 @@ func Test_RunPodSandbox_CPULimit_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorLimit: "9000", + annotations.ContainerProcessorLimit: "9000", }), ) runPodSandboxTest(t, request) @@ -423,7 +423,7 @@ func Test_RunPodSandbox_CPULimit_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorLimit: "90000", + annotations.ProcessorLimit: "90000", }), ) runPodSandboxTest(t, request) @@ -438,7 +438,7 @@ func Test_RunPodSandbox_CPULimit_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorLimit: "90000", + annotations.ProcessorLimit: "90000", }), ) runPodSandboxTest(t, request) @@ -453,7 +453,7 @@ func Test_RunPodSandbox_CPUWeight_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorWeight: "500", + annotations.ContainerProcessorWeight: "500", }), ) runPodSandboxTest(t, request) @@ -468,7 +468,7 @@ func Test_RunPodSandbox_CPUWeight_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorWeight: "500", + annotations.ContainerProcessorWeight: "500", }), ) runPodSandboxTest(t, request) @@ -483,7 +483,7 @@ func Test_RunPodSandbox_CPUWeight_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorWeight: "500", + annotations.ProcessorWeight: "500", }), ) runPodSandboxTest(t, request) @@ -498,7 +498,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s + annotations.ContainerStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }), ) runPodSandboxTest(t, request) @@ -513,7 +513,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s + annotations.StorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }), ) runPodSandboxTest(t, request) @@ -528,7 +528,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s + annotations.StorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }), ) runPodSandboxTest(t, request) @@ -543,7 +543,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerStorageQoSIopsMaximum: "300", + annotations.ContainerStorageQoSIopsMaximum: "300", }), ) runPodSandboxTest(t, request) @@ -558,7 +558,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSIopsMaximum: "300", + annotations.StorageQoSIopsMaximum: "300", }), ) runPodSandboxTest(t, request) @@ -573,7 +573,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSIopsMaximum: "300", + annotations.StorageQoSIopsMaximum: "300", }), ) runPodSandboxTest(t, request) @@ -588,7 +588,7 @@ func Test_RunPodSandbox_InitrdBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "initrd", + annotations.PreferredRootFSType: "initrd", }), ) runPodSandboxTest(t, request) @@ -603,7 +603,7 @@ func Test_RunPodSandbox_RootfsVhdBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "vhd", + annotations.PreferredRootFSType: "vhd", }), ) runPodSandboxTest(t, request) @@ -618,7 +618,7 @@ func Test_RunPodSandbox_VPCIEnabled_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationVPCIEnabled: "true", + annotations.VPCIEnabled: "true", }), ) runPodSandboxTest(t, request) @@ -633,7 +633,7 @@ func Test_RunPodSandbox_UEFIBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationKernelDirectBoot: "false", + annotations.KernelDirectBoot: "false", }), ) runPodSandboxTest(t, request) @@ -737,11 +737,11 @@ func Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } - output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annotations) + output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annots) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW", errorMsg, exitCode) @@ -781,12 +781,12 @@ func Test_RunPodSandbox_CustomizableScratchCustomSize_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb": "200", } - output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annotations) + output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annots) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW", errorMsg, exitCode) @@ -828,8 +828,8 @@ func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } mounts := []*runtime.Mount{ @@ -842,7 +842,7 @@ func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { "mount", } - output, errorMsg, exitCode := createSandboxContainerAndExec(t, annotations, mounts, cmd) + output, errorMsg, exitCode := createSandboxContainerAndExec(t, annots, mounts, cmd) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, t.Name()) @@ -1040,7 +1040,7 @@ func Test_RunPodSandbox_CPUGroup(t *testing.T) { Namespace: testNamespace, }, Annotations: map[string]string{ - oci.AnnotationCPUGroupID: presentID, + annotations.CPUGroupID: presentID, }, }, RuntimeHandler: test.runtimeHandler, @@ -1067,8 +1067,8 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } // Create a temporary ext4 VHD to mount into the container. @@ -1089,7 +1089,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { }, } - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sbRequest) defer removePodSandbox(t, client, ctx, podID) @@ -1107,7 +1107,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { // create 2 containers with vhd mounts and verify both can mount vhd for i := 1; i < 3; i++ { containerName := t.Name() + "-Container-" + strconv.Itoa(i) - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1126,7 +1126,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { // at the same time containers in a pod that don't have any mounts mounts = []*runtime.Mount{} containerName := t.Name() + "-Container-3" - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1280,8 +1280,8 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } vhdHostDir, err := ioutil.TempDir("", "") @@ -1305,7 +1305,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { }, } - sbRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(annotations)) + sbRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sbRequest) defer removePodSandbox(t, client, ctx, podID) @@ -1327,7 +1327,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { // create 2 containers with vhd mounts and verify both can mount vhd for i := 1; i < 3; i++ { containerName := t.Name() + "-Container-" + strconv.Itoa(i) - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1347,7 +1347,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { // at the same time containers in a pod that don't have any mounts mounts = []*runtime.Mount{} containerName := t.Name() + "-Container-3" - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1374,7 +1374,7 @@ func Test_RunPodSandbox_ProcessDump_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessDumpLocation: "/coredumps/core", + annotations.ContainerProcessDumpLocation: "/coredumps/core", }), ) @@ -1389,8 +1389,8 @@ func Test_RunPodSandbox_ProcessDump_LCOW(t *testing.T) { }, } - annotations := map[string]string{ - oci.AnnotationRLimitCore: "18446744073709551615;18446744073709551615", + annots := map[string]string{ + annotations.RLimitCore: "18446744073709551615;18446744073709551615", } // Setup container 1 that uses an image that stackoverflows shortly after starting. @@ -1406,7 +1406,7 @@ func Test_RunPodSandbox_ProcessDump_LCOW(t *testing.T) { Command: []string{ "./stackoverflow", }, - Annotations: annotations, + Annotations: annots, Mounts: mounts, }, PodSandboxId: podID, @@ -1485,7 +1485,7 @@ func Test_RunPodSandbox_ProcessDump_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisor19041RuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessDumpLocation: "C:\\processdump", + annotations.ContainerProcessDumpLocation: "C:\\processdump", }), ) @@ -1589,18 +1589,18 @@ func Test_RunPodSandbox_ProcessDump_WCOW_Hypervisor(t *testing.T) { } } -func createSandboxContainerAndExecForCustomScratch(t *testing.T, annotations map[string]string) (string, string, int) { +func createSandboxContainerAndExecForCustomScratch(t *testing.T, annots map[string]string) (string, string, int) { cmd := []string{ "df", } - return createSandboxContainerAndExec(t, annotations, nil, cmd) + return createSandboxContainerAndExec(t, annots, nil, cmd) } func createContainerInSandbox(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, podId, containerName, imageName string, command []string, - annotations map[string]string, mounts []*runtime.Mount, podConfig *runtime.PodSandboxConfig) string { + annots map[string]string, mounts []*runtime.Mount, podConfig *runtime.PodSandboxConfig) string { cRequest := getCreateContainerRequest(podId, containerName, imageName, command, podConfig) - cRequest.Config.Annotations = annotations + cRequest.Config.Annotations = annots cRequest.Config.Mounts = mounts containerID := createContainer(t, client, ctx, cRequest) @@ -1623,12 +1623,12 @@ func execContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx contex return output, errorMsg, exitCode } -func createSandboxContainerAndExec(t *testing.T, annotations map[string]string, mounts []*runtime.Mount, execCommand []string) (output string, errorMsg string, exitCode int) { +func createSandboxContainerAndExec(t *testing.T, annots map[string]string, mounts []*runtime.Mount, execCommand []string) (output string, errorMsg string, exitCode int) { client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sbRequest) defer removePodSandbox(t, client, ctx, podID) @@ -1652,7 +1652,7 @@ func createSandboxContainerAndExec(t *testing.T, annotations map[string]string, Command: []string{ "top", }, - Annotations: annotations, + Annotations: annots, Mounts: testMounts, }, PodSandboxId: podID, @@ -1685,14 +1685,14 @@ func Test_RunPodSandbox_KernelOptions_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - annotations := map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", - oci.AnnotationMemorySizeInMB: "2048", - oci.AnnotationKernelBootOptions: "hugepagesz=2M hugepages=10", + annots := map[string]string{ + annotations.FullyPhysicallyBacked: "true", + annotations.MemorySizeInMB: "2048", + annotations.KernelBootOptions: "hugepagesz=2M hugepages=10", } hugePagesCmd := []string{"grep", "-i", "HugePages_Total", "/proc/meminfo"} - output, errorMsg, exitCode := createSandboxContainerAndExec(t, annotations, nil, hugePagesCmd) + output, errorMsg, exitCode := createSandboxContainerAndExec(t, annots, nil, hugePagesCmd) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, t.Name()) diff --git a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go index 6be4ace5c5..49d1c7fab8 100644 --- a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go +++ b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" criruntime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -34,7 +34,7 @@ func Test_Scale_CPU_Limits_To_Sandbox(t *testing.T) { // set the limit to (roughly) 1 processor processorLimit := 10000 / runtime.NumCPU() contReq.Config.Annotations = map[string]string{ - oci.AnnotationContainerProcessorLimit: strconv.Itoa(processorLimit), + annotations.ContainerProcessorLimit: strconv.Itoa(processorLimit), } contID := createContainer(t, client, ctx, contReq) diff --git a/test/cri-containerd/stats_test.go b/test/cri-containerd/stats_test.go index a950fba4b0..22197b5ce0 100644 --- a/test/cri-containerd/stats_test.go +++ b/test/cri-containerd/stats_test.go @@ -4,10 +4,10 @@ package cri_containerd import ( "context" + "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "testing" - "github.com/Microsoft/hcsshim/internal/oci" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -369,9 +369,9 @@ func Test_SandboxStats_WorkingSet_PhysicallyBacked(t *testing.T) { t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "false", - oci.AnnotationEnableDeferredCommit: "false", - oci.AnnotationMemorySizeInMB: sizeInMBStr, + annotations.AllowOvercommit: "false", + annotations.EnableDeferredCommit: "false", + annotations.MemorySizeInMB: sizeInMBStr, }), ) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go index cd9ca03f7e..ae1dbdeab2 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -97,13 +98,13 @@ func verifyCloneContainerSpecs(templateSpec, cloneSpec *specs.Spec) error { } // for annotations check that the values of memory & cpu annotations are same - if templateSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] != cloneSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] { + if templateSpec.Annotations[annotations.ContainerMemorySizeInMB] != cloneSpec.Annotations[annotations.ContainerMemorySizeInMB] { return fmt.Errorf("memory size limit for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorCount] != cloneSpec.Annotations[oci.AnnotationContainerProcessorCount] { + if templateSpec.Annotations[annotations.ContainerProcessorCount] != cloneSpec.Annotations[annotations.ContainerProcessorCount] { return fmt.Errorf("processor count for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorLimit] != cloneSpec.Annotations[oci.AnnotationContainerProcessorLimit] { + if templateSpec.Annotations[annotations.ContainerProcessorLimit] != cloneSpec.Annotations[annotations.ContainerProcessorLimit] { return fmt.Errorf("processor limit for template and clone containers can not be different") } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go index b007ffdd7d..cccac4397b 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -24,8 +25,8 @@ const deviceUtilExeName = "device-util.exe" // getSpecKernelDrivers gets any device drivers specified on the spec. // Drivers are optional, therefore do not return an error if none are on the spec. -func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { - drivers := oci.ParseAnnotationCommaSeparated(oci.AnnotationVirtualMachineKernelDrivers, annotations) +func getSpecKernelDrivers(annots map[string]string) ([]string, error) { + drivers := oci.ParseAnnotationCommaSeparated(annotations.VirtualMachineKernelDrivers, annots) for _, driver := range drivers { if _, err := os.Stat(driver); err != nil { return nil, errors.Wrapf(err, "failed to find path to drivers at %s", driver) @@ -36,8 +37,8 @@ func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { // getDeviceExtensionPaths gets any device extensions paths specified on the spec. // device extensions are optional, therefore if none are on the spec, do not return an error. -func getDeviceExtensionPaths(annotations map[string]string) ([]string, error) { - extensions := oci.ParseAnnotationCommaSeparated(oci.AnnotationDeviceExtensions, annotations) +func getDeviceExtensionPaths(annots map[string]string) ([]string, error) { + extensions := oci.ParseAnnotationCommaSeparated(annotations.DeviceExtensions, annots) for _, ext := range extensions { if _, err := os.Stat(ext); err != nil { return nil, errors.Wrapf(err, "failed to find path to driver extensions at %s", ext) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go index 54c4be1eaf..8f51420540 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/uvmfolder" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -113,17 +114,17 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount // exclusive. func ConvertCPULimits(ctx context.Context, cid string, spec *specs.Spec, maxCPUCount int32) (int32, int32, int32, error) { cpuNumSet := 0 - cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, oci.AnnotationContainerProcessorCount, 0) + cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, annotations.ContainerProcessorCount, 0) if cpuCount > 0 { cpuNumSet++ } - cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, oci.AnnotationContainerProcessorLimit, 0) + cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, annotations.ContainerProcessorLimit, 0) if cpuLimit > 0 { cpuNumSet++ } - cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, oci.AnnotationContainerProcessorWeight, 0) + cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, annotations.ContainerProcessorWeight, 0) if cpuWeight > 0 { cpuNumSet++ } @@ -244,7 +245,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Memory Resources - memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, oci.AnnotationContainerMemorySizeInMB, 0) + memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, annotations.ContainerMemorySizeInMB, 0) if memoryMaxInMB > 0 { v1.MemoryMaximumInMB = int64(memoryMaxInMB) v2Container.Memory = &hcsschema.Memory{ @@ -253,8 +254,8 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Storage Resources - storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, oci.AnnotationContainerStorageQoSBandwidthMaximum, 0) - storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, oci.AnnotationContainerStorageQoSIopsMaximum, 0) + storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, annotations.ContainerStorageQoSBandwidthMaximum, 0) + storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, annotations.ContainerStorageQoSIopsMaximum, 0) if storageBandwidthMax > 0 || storageIopsMax > 0 { v1.StorageBandwidthMaximum = uint64(storageBandwidthMax) v1.StorageIOPSMaximum = uint64(storageIopsMax) @@ -392,7 +393,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter dumpPath = coi.HostingSystem.ProcessDumpLocation() } - if specDumpPath, ok := coi.Spec.Annotations[oci.AnnotationContainerProcessDumpLocation]; ok { + if specDumpPath, ok := coi.Spec.Annotations[annotations.ContainerProcessDumpLocation]; ok { // If a process dump path was specified at pod creation time for a hypervisor isolated pod, then // use this value. If one was specified on the container creation document then override with this // instead. Unlike Linux, Windows containers can set the dump path on a per container basis. @@ -466,8 +467,8 @@ func parseAssignedDevices(ctx context.Context, coi *createOptionsInternal, v2 *h // corresponding value the registry expects to be set. // // See DumpType at https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps for the mappings -func parseDumpType(annotations map[string]string) (int32, error) { - dmpTypeStr := annotations[oci.AnnotationWCOWProcessDumpType] +func parseDumpType(annots map[string]string) (int32, error) { + dmpTypeStr := annots[annotations.WCOWProcessDumpType] switch dmpTypeStr { case "": // If no type specified, default to full dumps. diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go index 1bdf795ae1..c4fec89f27 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go @@ -14,9 +14,9 @@ import ( "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -25,7 +25,7 @@ import ( // shim option is set. Right now we only support Nvidia gpus, so this will default to // a gpu vhd with nvidia files func getGPUVHDPath(coi *createOptionsInternal) (string, error) { - gpuVHDPath, ok := coi.Spec.Annotations[oci.AnnotationGPUVHDPath] + gpuVHDPath, ok := coi.Spec.Annotations[annotations.GPUVHDPath] if !ok || gpuVHDPath == "" { return "", fmt.Errorf("no gpu vhd specified %s", gpuVHDPath) } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go deleted file mode 100644 index bed5230470..0000000000 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go +++ /dev/null @@ -1,237 +0,0 @@ -package oci - -const ( - // AnnotationContainerMemorySizeInMB overrides the container memory size set - // via the OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Memory.Limit`. - AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" - - // AnnotationContainerProcessorCount overrides the container processor count - // set via the OCI spec. - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. - AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" - - // AnnotationContainerProcessorLimit overrides the container processor limit - // set via the OCI spec. - // - // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.CPU.Maximum`. - AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" - - // AnnotationContainerProcessorWeight overrides the container processor - // weight set via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. - AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" - - // AnnotationContainerStorageQoSBandwidthMaximum overrides the container - // storage bandwidth per second set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Bps`. - AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" - - // AnnotationContainerStorageQoSIopsMaximum overrides the container storage - // maximum iops set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Iops`. - AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" - - // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd - AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" - - // AnnotationVirtualMachineKernelDrivers indicates what drivers to install in the pod. - // This value should contain a list of comma separated directories containing all - // files and information needed to install given driver(s). For windows, this may - // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. - // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. - AnnotationVirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" - - // AnnotationDeviceExtensions contains a comma separated list of full paths to device extension files. - // The content of these are added to a container's hcs create document. - AnnotationDeviceExtensions = "io.microsoft.container.wcow.deviceextensions" - - // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process - // container as and instead inherit the user token from the executable that is launching the container process. - AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" - - // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). - AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" - - // AnnotationAllowOvercommit indicates if we should allow over commit memory for UVM. - // Defaults to true. For physical backed memory, set to false. - AnnotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" - - // AnnotationEnableDeferredCommit indicates if we should allow deferred memory commit for UVM. - // Defaults to false. For virtual memory with deferred commit, set to true. - AnnotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" - - // AnnotationEnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM - // to trim non-zeroed pages from the working set (if supported by the guest operating system). - AnnotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" - - // AnnotationMemorySizeInMB overrides the container memory size set via the - // OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - AnnotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" - - // AnnotationMemoryLowMMIOGapInMB indicates the low MMIO gap in MB - AnnotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO base in MB - AnnotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO gap in MB - AnnotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" - - // annotationProcessorCount overrides the hypervisor isolated vCPU count set - // via the OCI spec. - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" - - // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set - // via the OCI spec. - // - // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" - - // AnnotationProcessorWeight overrides the hypervisor isolated vCPU weight set - // via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" - - // AnnotationVPMemCount indicates the max number of vpmem devices that can be used on the UVM - AnnotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" - - // AnnotationVPMemSize indicates the size of the VPMem devices. - AnnotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" - - // AnnotationPreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. - // valid values are "initrd" or "vhd" - AnnotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" - - // AnnotationBootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM - AnnotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" - - // AnnotationKernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` - AnnotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" - - // AnnotationVPCIEnabled indicates that pci support should be enabled for the LCOW UVM - AnnotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" - - // AnnotationVPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping - AnnotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" - - // AnnotationKernelBootOptions is used to specify kernel options used while booting a linux kernel - AnnotationKernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" - - // AnnotationStorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` - // will default to the platform default. - AnnotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" - - // AnnotationStorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will - // default to the platform default. - AnnotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" - - // AnnotationFullyPhysicallyBacked indicates that the UVM should use physically backed memory only, - // including for additional devices added later. - AnnotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" - - // AnnotationDisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM - // for WCOW. - AnnotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" - - // AnnotationVSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM - AnnotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" - - // AnnotationCPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any - AnnotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" - - // AnnotationSaveAsTemplate annotation must be used with a pod & container creation request. - // If this annotation is present in the request then it will save the UVM (pod) - // and the container(s) inside it as a template. However, this also means that this - // pod and the containers inside this pod will permananetly stay in the - // paused/templated state and can not be resumed again. - AnnotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" - - // AnnotationTemplateID should be used when creating a pod or a container from a template. - // When creating a pod from a template use the ID of the templated pod as the - // TemplateID and when creating a container use the ID of the templated container as - // the TemplateID. It is the client's responsibility to make sure that the sandbox - // within which a cloned container needs to be created must also be created from the - // same template. - AnnotationTemplateID = "io.microsoft.virtualmachine.templateid" - - // AnnotationNetworkConfigProxy holds the address of the network config proxy service. - // If set, network setup will be attempted via ncproxy. - AnnotationNetworkConfigProxy = "io.microsoft.network.ncproxy" - - // AnnotationNcproxyContainerID indicates whether or not to use the hcsshim container ID - // when setting up ncproxy and computeagent - AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" - - // AnnotationEncryptedScratchDisk indicates whether or not the container scratch disks - // should be encrypted or not - AnnotationEncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" - - // AnnotationSecurityPolicy is used to specify a security policy for opengcs to enforce - AnnotationSecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" - // AnnotationContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As - // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of - // some kind (vhd, bind mount, fileshare mount etc.) - AnnotationContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" - - // AnnotationWCOWProcessDumpType specifies the type of dump to create when generating a local user mode - // process dump for Windows containers. The supported options are "mini", and "full". - // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps - AnnotationWCOWProcessDumpType = "io.microsoft.wcow.processdumptype" - - // AnnotationRLimitCore specifies the core rlimit value for a container. This will need to be set - // in order to have core dumps generated for a given container. - AnnotationRLimitCore = "io.microsoft.lcow.rlimitcore" -) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go index 3c3f89280d..569b035654 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go @@ -2,28 +2,22 @@ package oci import ( "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" ) -// KubernetesContainerTypeAnnotation is the annotation used by CRI to define the `ContainerType`. -const KubernetesContainerTypeAnnotation = "io.kubernetes.cri.container-type" - -// KubernetesSandboxIDAnnotation is the annotation used by CRI to define the -// KubernetesContainerTypeAnnotation == "sandbox"` ID. -const KubernetesSandboxIDAnnotation = "io.kubernetes.cri.sandbox-id" - // KubernetesContainerType defines the valid types of the -// `KubernetesContainerTypeAnnotation` annotation. +// `annotations.KubernetesContainerType` annotation. type KubernetesContainerType string const ( // KubernetesContainerTypeNone is only valid when - // `KubernetesContainerTypeAnnotation` is not set. + // `annotations.KubernetesContainerType` is not set. KubernetesContainerTypeNone KubernetesContainerType = "" // KubernetesContainerTypeContainer is valid when - // `KubernetesContainerTypeAnnotation == "container"`. + // `annotations.KubernetesContainerType == "container"`. KubernetesContainerTypeContainer KubernetesContainerType = "container" // KubernetesContainerTypeSandbox is valid when - // `KubernetesContainerTypeAnnotation == "sandbox"`. + // `annotations.KubernetesContainerType == "sandbox"`. KubernetesContainerTypeSandbox KubernetesContainerType = "sandbox" ) @@ -32,27 +26,27 @@ const ( // annotations and if found validates the set before returning. func GetSandboxTypeAndID(specAnnotations map[string]string) (KubernetesContainerType, string, error) { var ct KubernetesContainerType - if t, ok := specAnnotations[KubernetesContainerTypeAnnotation]; ok { + if t, ok := specAnnotations[annotations.KubernetesContainerType]; ok { switch t { case string(KubernetesContainerTypeContainer): ct = KubernetesContainerTypeContainer case string(KubernetesContainerTypeSandbox): ct = KubernetesContainerTypeSandbox default: - return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", KubernetesContainerTypeAnnotation, t) + return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", annotations.KubernetesContainerType, t) } } - id := specAnnotations[KubernetesSandboxIDAnnotation] + id := specAnnotations[annotations.KubernetesSandboxID] switch ct { case KubernetesContainerTypeContainer, KubernetesContainerTypeSandbox: if id == "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesContainerTypeAnnotation, KubernetesSandboxIDAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesContainerType, annotations.KubernetesSandboxID) } default: if id != "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesSandboxIDAnnotation, KubernetesContainerTypeAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesSandboxID, annotations.KubernetesContainerType) } } return ct, id, nil diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go index 85f52016ff..6452241eca 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go @@ -1,6 +1,9 @@ package oci -import "github.com/opencontainers/runtime-spec/specs-go" +import ( + "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/opencontainers/runtime-spec/specs-go" +) // IsLCOW checks if `s` is a LCOW config. func IsLCOW(s *specs.Spec) bool { @@ -19,5 +22,5 @@ func IsIsolated(s *specs.Spec) bool { // IsJobContainer checks if `s` is asking for a Windows job container. func IsJobContainer(s *specs.Spec) bool { - return s.Annotations[AnnotationHostProcessContainer] == "true" + return s.Annotations[annotations.HostProcessContainer] == "true" } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go index c77a90731a..fcf41fba08 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "strings" @@ -219,13 +220,13 @@ func parseAnnotationsString(a map[string]string, key string, def string) string // if this create request should be considered as a template creation request. If value // is found the returns the actual value, returns false otherwise. func ParseAnnotationsSaveAsTemplate(ctx context.Context, s *specs.Spec) bool { - return parseAnnotationsBool(ctx, s.Annotations, AnnotationSaveAsTemplate, false) + return parseAnnotationsBool(ctx, s.Annotations, annotations.SaveAsTemplate, false) } // ParseAnnotationsTemplateID searches for the templateID in the create request. If the // value is found then returns the value otherwise returns the empty string. func ParseAnnotationsTemplateID(ctx context.Context, s *specs.Spec) string { - return parseAnnotationsString(s.Annotations, AnnotationTemplateID, "") + return parseAnnotationsString(s.Annotations, annotations.TemplateID, "") } func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, templateID string, err error) { @@ -244,7 +245,7 @@ func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, // handleAnnotationKernelDirectBoot handles parsing annotationKernelDirectBoot and setting // implied annotations from the result. func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.KernelDirect = parseAnnotationsBool(ctx, a, AnnotationKernelDirectBoot, lopts.KernelDirect) + lopts.KernelDirect = parseAnnotationsBool(ctx, a, annotations.KernelDirectBoot, lopts.KernelDirect) if !lopts.KernelDirect { lopts.KernelFile = uvm.KernelFile } @@ -253,7 +254,7 @@ func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, // handleAnnotationPreferredRootFSType handles parsing annotationPreferredRootFSType and setting // implied annotations from the result func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, AnnotationPreferredRootFSType, lopts.PreferredRootFSType) + lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, annotations.PreferredRootFSType, lopts.PreferredRootFSType) switch lopts.PreferredRootFSType { case uvm.PreferredRootFSTypeInitRd: lopts.RootFSFile = uvm.InitrdFile @@ -267,7 +268,7 @@ func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]strin func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]string, opts interface{}) { switch options := opts.(type) { case *uvm.OptionsLCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false options.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd @@ -275,7 +276,7 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str options.VPMemDeviceCount = 0 } case *uvm.OptionsWCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false } @@ -285,8 +286,8 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str // handleCloneAnnotations handles parsing annotations related to template creation and cloning // Since late cloning is only supported for WCOW this function only deals with WCOW options. func handleCloneAnnotations(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) (err error) { - wopts.IsTemplate = parseAnnotationsBool(ctx, a, AnnotationSaveAsTemplate, false) - templateID := parseAnnotationsString(a, AnnotationTemplateID, "") + wopts.IsTemplate = parseAnnotationsBool(ctx, a, annotations.SaveAsTemplate, false) + templateID := parseAnnotationsString(a, annotations.TemplateID, "") if templateID != "" { tc, err := clone.FetchTemplateConfig(ctx, templateID) if err != nil { @@ -310,29 +311,29 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( } if IsLCOW(s) { lopts := uvm.NewDefaultOptionsLCOW(id, owner) - lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, lopts.MemorySizeInMB) - lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) - lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) - lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) - lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, lopts.AllowOvercommit) - lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, lopts.EnableDeferredCommit) - lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) - lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, lopts.ProcessorCount) - lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, lopts.ProcessorLimit) - lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, lopts.ProcessorWeight) - lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, AnnotationVPMemCount, lopts.VPMemDeviceCount) - lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, AnnotationVPMemSize, lopts.VPMemSizeBytes) - lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) - lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) - lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) - lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPCIEnabled, lopts.VPCIEnabled) - lopts.BootFilesPath = parseAnnotationsString(s.Annotations, AnnotationBootFilesRootPath, lopts.BootFilesPath) - lopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, lopts.CPUGroupID) - lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, lopts.NetworkConfigProxy) - lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, AnnotationEncryptedScratchDisk, lopts.EnableScratchEncryption) - lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, AnnotationSecurityPolicy, lopts.SecurityPolicy) - lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, AnnotationKernelBootOptions, lopts.KernelBootOptions) - lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, lopts.ProcessDumpLocation) + lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, lopts.MemorySizeInMB) + lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) + lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) + lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) + lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, lopts.AllowOvercommit) + lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, lopts.EnableDeferredCommit) + lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableColdDiscardHint, lopts.EnableColdDiscardHint) + lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, lopts.ProcessorCount) + lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, lopts.ProcessorLimit) + lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, lopts.ProcessorWeight) + lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, annotations.VPMemCount, lopts.VPMemDeviceCount) + lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, annotations.VPMemSize, lopts.VPMemSizeBytes) + lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, annotations.VPMemNoMultiMapping, lopts.VPMemNoMultiMapping) + lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) + lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) + lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotations.VPCIEnabled, lopts.VPCIEnabled) + lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotations.BootFilesRootPath, lopts.BootFilesPath) + lopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, lopts.CPUGroupID) + lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, lopts.NetworkConfigProxy) + lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, annotations.EncryptedScratchDisk, lopts.EnableScratchEncryption) + lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, annotations.SecurityPolicy, lopts.SecurityPolicy) + lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) + lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, lopts.ProcessDumpLocation) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) @@ -342,22 +343,22 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( return lopts, nil } else if IsWCOW(s) { wopts := uvm.NewDefaultOptionsWCOW(id, owner) - wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, wopts.MemorySizeInMB) - wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) - wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) - wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) - wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, wopts.AllowOvercommit) - wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, wopts.EnableDeferredCommit) - wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, wopts.ProcessorCount) - wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, wopts.ProcessorLimit) - wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, wopts.ProcessorWeight) - wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) - wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) - wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, AnnotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) - wopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, wopts.CPUGroupID) - wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, wopts.NetworkConfigProxy) - wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, AnnotationVSMBNoDirectMap, wopts.NoDirectMap) - wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, wopts.ProcessDumpLocation) + wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, wopts.MemorySizeInMB) + wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) + wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) + wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) + wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, wopts.AllowOvercommit) + wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, wopts.EnableDeferredCommit) + wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, wopts.ProcessorCount) + wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, wopts.ProcessorLimit) + wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, wopts.ProcessorWeight) + wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) + wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) + wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, annotations.DisableCompartmentNamespace, wopts.DisableCompartmentNamespace) + wopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, wopts.CPUGroupID) + wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, wopts.NetworkConfigProxy) + wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, annotations.VSMBNoDirectMap, wopts.NoDirectMap) + wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, wopts.ProcessDumpLocation) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) if err := handleCloneAnnotations(ctx, s.Annotations, wopts); err != nil { return nil, err @@ -374,24 +375,24 @@ func UpdateSpecFromOptions(s specs.Spec, opts *runhcsopts.Options) specs.Spec { return s } - if _, ok := s.Annotations[AnnotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { - s.Annotations[AnnotationBootFilesRootPath] = opts.BootFilesRootPath + if _, ok := s.Annotations[annotations.BootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { + s.Annotations[annotations.BootFilesRootPath] = opts.BootFilesRootPath } - if _, ok := s.Annotations[AnnotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { - s.Annotations[AnnotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) + if _, ok := s.Annotations[annotations.ProcessorCount]; !ok && opts.VmProcessorCount != 0 { + s.Annotations[annotations.ProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) } - if _, ok := s.Annotations[AnnotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { - s.Annotations[AnnotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) + if _, ok := s.Annotations[annotations.MemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { + s.Annotations[annotations.MemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) } - if _, ok := s.Annotations[AnnotationGPUVHDPath]; !ok && opts.GPUVHDPath != "" { - s.Annotations[AnnotationGPUVHDPath] = opts.GPUVHDPath + if _, ok := s.Annotations[annotations.GPUVHDPath]; !ok && opts.GPUVHDPath != "" { + s.Annotations[annotations.GPUVHDPath] = opts.GPUVHDPath } - if _, ok := s.Annotations[AnnotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { - s.Annotations[AnnotationNetworkConfigProxy] = opts.NCProxyAddr + if _, ok := s.Annotations[annotations.NetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { + s.Annotations[annotations.NetworkConfigProxy] = opts.NCProxyAddr } return s diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go new file mode 100644 index 0000000000..0042b3d24c --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go @@ -0,0 +1,253 @@ +package annotations + +const ( + // ContainerMemorySizeInMB overrides the container memory size set + // via the OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Memory.Limit`. + ContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" + + // ContainerProcessorCount overrides the container processor count + // set via the OCI spec. + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. + ContainerProcessorCount = "io.microsoft.container.processor.count" + + // ContainerProcessorLimit overrides the container processor limit + // set via the OCI spec. + // + // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.CPU.Maximum`. + ContainerProcessorLimit = "io.microsoft.container.processor.limit" + + // ContainerProcessorWeight overrides the container processor + // weight set via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. + ContainerProcessorWeight = "io.microsoft.container.processor.weight" + + // ContainerStorageQoSBandwidthMaximum overrides the container + // storage bandwidth per second set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Bps`. + ContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" + + // ContainerStorageQoSIopsMaximum overrides the container storage + // maximum iops set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Iops`. + ContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" + + // GPUVHDPath overrides the default path to search for the gpu vhd + GPUVHDPath = "io.microsoft.lcow.gpuvhdpath" + + // ContainerGPUCapabilities is used to find the gpu capabilities on the container spec + ContainerGPUCapabilities = "io.microsoft.container.gpu.capabilities" + + // VirtualMachineKernelDrivers indicates what drivers to install in the pod. + // This value should contain a list of comma separated directories containing all + // files and information needed to install given driver(s). For windows, this may + // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. + // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. + VirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" + + // DeviceExtensions contains a comma separated list of full paths to device extension files. + // The content of these are added to a container's hcs create document. + DeviceExtensions = "io.microsoft.container.wcow.deviceextensions" + + // HostProcessInheritUser indicates whether to ignore the username passed in to run a host process + // container as and instead inherit the user token from the executable that is launching the container process. + HostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" + + // HostProcessContainer indicates to launch a host process container (job container in this repository). + HostProcessContainer = "microsoft.com/hostprocess-container" + + // AllowOvercommit indicates if we should allow over commit memory for UVM. + // Defaults to true. For physical backed memory, set to false. + AllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" + + // EnableDeferredCommit indicates if we should allow deferred memory commit for UVM. + // Defaults to false. For virtual memory with deferred commit, set to true. + EnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" + + // EnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM + // to trim non-zeroed pages from the working set (if supported by the guest operating system). + EnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" + + // MemorySizeInMB overrides the container memory size set via the + // OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + MemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" + + // MemoryLowMMIOGapInMB indicates the low MMIO gap in MB + MemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO base in MB + MemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO gap in MB + MemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" + + // ProcessorCount overrides the hypervisor isolated vCPU count set + // via the OCI spec. + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" + + // ProcessorLimit overrides the hypervisor isolated vCPU limit set + // via the OCI spec. + // + // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" + + // ProcessorWeight overrides the hypervisor isolated vCPU weight set + // via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" + + // VPMemCount indicates the max number of vpmem devices that can be used on the UVM + VPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" + + // VPMemSize indicates the size of the VPMem devices. + VPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" + + // PreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. + // valid values are "initrd" or "vhd" + PreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" + + // BootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM + BootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" + + // KernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` + KernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" + + // VPCIEnabled indicates that pci support should be enabled for the LCOW UVM + VPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" + + // VPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping + VPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" + + // KernelBootOptions is used to specify kernel options used while booting a linux kernel + KernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" + + // StorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` + // will default to the platform default. + StorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" + + // StorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will + // default to the platform default. + StorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" + + // FullyPhysicallyBacked indicates that the UVM should use physically backed memory only, + // including for additional devices added later. + FullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" + + // DisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM + // for WCOW. + DisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" + + // VSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM + VSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" + + // CPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any + CPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" + + // SaveAsTemplate annotation must be used with a pod & container creation request. + // If this annotation is present in the request then it will save the UVM (pod) + // and the container(s) inside it as a template. However, this also means that this + // pod and the containers inside this pod will permananetly stay in the + // paused/templated state and can not be resumed again. + SaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" + + // TemplateID should be used when creating a pod or a container from a template. + // When creating a pod from a template use the ID of the templated pod as the + // TemplateID and when creating a container use the ID of the templated container as + // the TemplateID. It is the client's responsibility to make sure that the sandbox + // within which a cloned container needs to be created must also be created from the + // same template. + TemplateID = "io.microsoft.virtualmachine.templateid" + + // NetworkConfigProxy holds the address of the network config proxy service. + // If set, network setup will be attempted via ncproxy. + NetworkConfigProxy = "io.microsoft.network.ncproxy" + + // NcproxyContainerID indicates whether or not to use the hcsshim container ID + // when setting up ncproxy and computeagent + NcproxyContainerID = "io.microsoft.network.ncproxy.containerid" + + // EncryptedScratchDisk indicates whether or not the container scratch disks + // should be encrypted or not + EncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + + // SecurityPolicy is used to specify a security policy for opengcs to enforce + SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // ContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As + // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of + // some kind (vhd, bind mount, fileshare mount etc.) + ContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" + + // WCOWProcessDumpType specifies the type of dump to create when generating a local user mode + // process dump for Windows containers. The supported options are "mini", and "full". + // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps + WCOWProcessDumpType = "io.microsoft.wcow.processdumptype" + + // RLimitCore specifies the core rlimit value for a container. This will need to be set + // in order to have core dumps generated for a given container. + RLimitCore = "io.microsoft.lcow.rlimitcore" + + // LCOWDevShmSizeInKb specifies the size of LCOW /dev/shm. + LCOWDevShmSizeInKb = "io.microsoft.lcow.shm.size-kb" + + // LCOWPrivileged is used to specify that the container should be run in privileged mode + LCOWPrivileged = "io.microsoft.virtualmachine.lcow.privileged" + + // KubernetesContainerType is the annotation used by CRI to define the `ContainerType`. + KubernetesContainerType = "io.kubernetes.cri.container-type" + + // KubernetesSandboxID is the annotation used by CRI to define the + // KubernetesContainerType == "sandbox"` ID. + KubernetesSandboxID = "io.kubernetes.cri.sandbox-id" +) diff --git a/test/vendor/modules.txt b/test/vendor/modules.txt index 40a7a888f4..27c7647555 100644 --- a/test/vendor/modules.txt +++ b/test/vendor/modules.txt @@ -63,6 +63,7 @@ github.com/Microsoft/hcsshim/internal/wclayer github.com/Microsoft/hcsshim/internal/wcow github.com/Microsoft/hcsshim/internal/winapi github.com/Microsoft/hcsshim/osversion +github.com/Microsoft/hcsshim/pkg/annotations github.com/Microsoft/hcsshim/pkg/go-runhcs github.com/Microsoft/hcsshim/pkg/ociwclayer github.com/Microsoft/hcsshim/pkg/octtrpc