diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index abd9cbc506..6ed3c35b8b 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" eventstypes "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/runtime" @@ -67,7 +68,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation: '%s': '%s' got '%s'", - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesContainerType, oci.KubernetesContainerTypeSandbox, ct) } @@ -75,7 +76,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation '%s': '%s' got '%s'", - oci.KubernetesSandboxIDAnnotation, + annotations.KubernetesSandboxID, req.ID, sid) } @@ -175,7 +176,7 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques p.host = parent if parent != nil { cid := req.ID - if id, ok := s.Annotations[oci.AnnotationNcproxyContainerID]; ok { + if id, ok := s.Annotations[annotations.NcproxyContainerID]; ok { cid = id } caAddr := fmt.Sprintf(uvm.ComputeAgentAddrFmt, cid) @@ -319,7 +320,7 @@ func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *sp return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation: '%s': '%s' got '%s'", - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesContainerType, oci.KubernetesContainerTypeContainer, ct) } @@ -327,7 +328,7 @@ func (p *pod) CreateTask(ctx context.Context, req *task.CreateTaskRequest, s *sp return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "expected annotation '%s': '%s' got '%s'", - oci.KubernetesSandboxIDAnnotation, + annotations.KubernetesSandboxID, p.id, sid) } diff --git a/cmd/containerd-shim-runhcs-v1/start.go b/cmd/containerd-shim-runhcs-v1/start.go index e5fa54929a..520ddcb7f3 100644 --- a/cmd/containerd-shim-runhcs-v1/start.go +++ b/cmd/containerd-shim-runhcs-v1/start.go @@ -12,6 +12,7 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/containerd/containerd/runtime/v2/shim" "github.com/containerd/containerd/runtime/v2/task" "github.com/containerd/ttrpc" @@ -102,8 +103,8 @@ The start command can either start a new shim or return an address to an existin if isSandbox && idFlag != sbid { return errors.Errorf( "'id' and '%s' must match for '%s=%s'", - oci.KubernetesSandboxIDAnnotation, - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesSandboxID, + annotations.KubernetesContainerType, oci.KubernetesContainerTypeSandbox) } diff --git a/cmd/containerd-shim-runhcs-v1/task_hcs.go b/cmd/containerd-shim-runhcs-v1/task_hcs.go index 9f6f5a9a11..4ecb35716f 100644 --- a/cmd/containerd-shim-runhcs-v1/task_hcs.go +++ b/cmd/containerd-shim-runhcs-v1/task_hcs.go @@ -37,6 +37,7 @@ import ( "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" ) const bytesPerMB = 1024 * 1024 @@ -52,7 +53,7 @@ func newHcsStandaloneTask(ctx context.Context, events publisher, req *task.Creat return nil, errors.Wrapf( errdefs.ErrFailedPrecondition, "cannot create standalone task, expected no annotation: '%s': got '%s'", - oci.KubernetesContainerTypeAnnotation, + annotations.KubernetesContainerType, ct) } diff --git a/cmd/runhcs/container.go b/cmd/runhcs/container.go index 66b0bb9589..6726606839 100644 --- a/cmd/runhcs/container.go +++ b/cmd/runhcs/container.go @@ -23,6 +23,7 @@ import ( "github.com/Microsoft/hcsshim/internal/runhcs" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "golang.org/x/sys/windows" @@ -207,8 +208,8 @@ func launchShim(cmd, pidFile, logFile string, args []string, data interface{}) ( // returns `("", false)`. func parseSandboxAnnotations(a map[string]string) (string, bool) { var t, id string - if t = a["io.kubernetes.cri.container-type"]; t != "" { - id = a["io.kubernetes.cri.sandbox-id"] + if t = a[annotations.KubernetesContainerType]; t != "" { + id = a[annotations.KubernetesSandboxID] } else if t = a["io.kubernetes.cri-o.ContainerType"]; t != "" { id = a["io.kubernetes.cri-o.SandboxID"] } else if t = a["io.kubernetes.docker.type"]; t != "" { diff --git a/internal/guest/runtime/hcsv2/nvidia_utils.go b/internal/guest/runtime/hcsv2/nvidia_utils.go index 09931368d2..206eb4fade 100644 --- a/internal/guest/runtime/hcsv2/nvidia_utils.go +++ b/internal/guest/runtime/hcsv2/nvidia_utils.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/cmd/gcstools/generichook" "github.com/Microsoft/hcsshim/internal/guest/storage/pci" + "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -19,9 +20,6 @@ import ( // this MUST match the path mapped to in the shim const lcowNvidiaMountPath = "/run/nvidia" -// annotation to find the gpu capabilities on the container spec -// must match the hcsshim annotation string for gpu capabilities -const annotationContainerGPUCapabilities = "io.microsoft.container.gpu.capabilities" const nvidiaDebugFilePath = "/nvidia-container.log" const nvidiaToolBinary = "nvidia-container-cli" @@ -50,7 +48,7 @@ func addNvidiaDevicePreHook(ctx context.Context, spec *oci.Spec) error { "configure", "--ldconfig=@/sbin/ldconfig", } - if capabilities, ok := spec.Annotations[annotationContainerGPUCapabilities]; ok { + if capabilities, ok := spec.Annotations[annotations.ContainerGPUCapabilities]; ok { caps := strings.Split(capabilities, ",") for _, c := range caps { args = append(args, fmt.Sprintf("--%s", c)) diff --git a/internal/guest/runtime/hcsv2/sandbox_container.go b/internal/guest/runtime/hcsv2/sandbox_container.go index a4bd594505..14e3366003 100644 --- a/internal/guest/runtime/hcsv2/sandbox_container.go +++ b/internal/guest/runtime/hcsv2/sandbox_container.go @@ -11,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/guest/network" "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" @@ -112,7 +113,7 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) ( } } - if rlimCore := spec.Annotations["io.microsoft.lcow.rlimitcore"]; rlimCore != "" { + if rlimCore := spec.Annotations[annotations.RLimitCore]; rlimCore != "" { if err := setCoreRLimit(spec, rlimCore); err != nil { return err } diff --git a/internal/guest/runtime/hcsv2/spec.go b/internal/guest/runtime/hcsv2/spec.go index 99373ba75b..d53fa576ba 100644 --- a/internal/guest/runtime/hcsv2/spec.go +++ b/internal/guest/runtime/hcsv2/spec.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/user" oci "github.com/opencontainers/runtime-spec/specs-go" @@ -197,7 +198,7 @@ func getGroup(spec *oci.Spec, filter func(user.Group) bool) (user.Group, error) // applyAnnotationsToSpec modifies the spec based on additional information from annotations func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error { // Check if we need to override container's /dev/shm - if val, ok := spec.Annotations["io.microsoft.container.storage.shm.size-kb"]; ok { + if val, ok := spec.Annotations[annotations.LCOWDevShmSizeInKb]; ok { sz, err := strconv.ParseInt(val, 10, 64) if err != nil { return errors.Wrap(err, "/dev/shm size must be a valid integer") @@ -220,8 +221,8 @@ func applyAnnotationsToSpec(ctx context.Context, spec *oci.Spec) error { } // Check if we need to do any capability/device mappings - if spec.Annotations["io.microsoft.virtualmachine.lcow.privileged"] == "true" { - log.G(ctx).Debug("'io.microsoft.virtualmachine.lcow.privileged' set for privileged container") + if spec.Annotations[annotations.LCOWPrivileged] == "true" { + log.G(ctx).Debugf("'%s' set for privileged container", annotations.LCOWPrivileged) // Add all host devices hostDevices, err := devices.HostDevices() diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 634c94678f..d58d986057 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -26,6 +26,7 @@ import ( "github.com/Microsoft/hcsshim/internal/guest/storage/pmem" "github.com/Microsoft/hcsshim/internal/guest/storage/scsi" "github.com/Microsoft/hcsshim/internal/guest/transport" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/pkg/securitypolicy" shellwords "github.com/mattn/go-shellwords" "github.com/pkg/errors" @@ -156,7 +157,7 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM } var namespaceID string - criType, isCRI := settings.OCISpecification.Annotations["io.kubernetes.cri.container-type"] + criType, isCRI := settings.OCISpecification.Annotations[annotations.KubernetesContainerType] if isCRI { switch criType { case "sandbox": @@ -180,7 +181,7 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM return nil, err } case "container": - sid, ok := settings.OCISpecification.Annotations["io.kubernetes.cri.sandbox-id"] + sid, ok := settings.OCISpecification.Annotations[annotations.KubernetesSandboxID] if !ok || sid == "" { return nil, errors.Errorf("unsupported 'io.kubernetes.cri.sandbox-id': '%s'", sid) } diff --git a/internal/guest/runtime/hcsv2/workload_container.go b/internal/guest/runtime/hcsv2/workload_container.go index e85311fc59..913552fc3d 100644 --- a/internal/guest/runtime/hcsv2/workload_container.go +++ b/internal/guest/runtime/hcsv2/workload_container.go @@ -10,6 +10,7 @@ import ( "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runc/libcontainer/devices" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -161,7 +162,7 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci. return err } - if rlimCore := spec.Annotations["io.microsoft.lcow.rlimitcore"]; rlimCore != "" { + if rlimCore := spec.Annotations[annotations.RLimitCore]; rlimCore != "" { if err := setCoreRLimit(spec, rlimCore); err != nil { return err } diff --git a/internal/hcsoci/create.go b/internal/hcsoci/create.go index cd9ca03f7e..ae1dbdeab2 100644 --- a/internal/hcsoci/create.go +++ b/internal/hcsoci/create.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -97,13 +98,13 @@ func verifyCloneContainerSpecs(templateSpec, cloneSpec *specs.Spec) error { } // for annotations check that the values of memory & cpu annotations are same - if templateSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] != cloneSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] { + if templateSpec.Annotations[annotations.ContainerMemorySizeInMB] != cloneSpec.Annotations[annotations.ContainerMemorySizeInMB] { return fmt.Errorf("memory size limit for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorCount] != cloneSpec.Annotations[oci.AnnotationContainerProcessorCount] { + if templateSpec.Annotations[annotations.ContainerProcessorCount] != cloneSpec.Annotations[annotations.ContainerProcessorCount] { return fmt.Errorf("processor count for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorLimit] != cloneSpec.Annotations[oci.AnnotationContainerProcessorLimit] { + if templateSpec.Annotations[annotations.ContainerProcessorLimit] != cloneSpec.Annotations[annotations.ContainerProcessorLimit] { return fmt.Errorf("processor limit for template and clone containers can not be different") } diff --git a/internal/hcsoci/devices.go b/internal/hcsoci/devices.go index b007ffdd7d..cccac4397b 100644 --- a/internal/hcsoci/devices.go +++ b/internal/hcsoci/devices.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -24,8 +25,8 @@ const deviceUtilExeName = "device-util.exe" // getSpecKernelDrivers gets any device drivers specified on the spec. // Drivers are optional, therefore do not return an error if none are on the spec. -func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { - drivers := oci.ParseAnnotationCommaSeparated(oci.AnnotationVirtualMachineKernelDrivers, annotations) +func getSpecKernelDrivers(annots map[string]string) ([]string, error) { + drivers := oci.ParseAnnotationCommaSeparated(annotations.VirtualMachineKernelDrivers, annots) for _, driver := range drivers { if _, err := os.Stat(driver); err != nil { return nil, errors.Wrapf(err, "failed to find path to drivers at %s", driver) @@ -36,8 +37,8 @@ func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { // getDeviceExtensionPaths gets any device extensions paths specified on the spec. // device extensions are optional, therefore if none are on the spec, do not return an error. -func getDeviceExtensionPaths(annotations map[string]string) ([]string, error) { - extensions := oci.ParseAnnotationCommaSeparated(oci.AnnotationDeviceExtensions, annotations) +func getDeviceExtensionPaths(annots map[string]string) ([]string, error) { + extensions := oci.ParseAnnotationCommaSeparated(annotations.DeviceExtensions, annots) for _, ext := range extensions { if _, err := os.Stat(ext); err != nil { return nil, errors.Wrapf(err, "failed to find path to driver extensions at %s", ext) diff --git a/internal/hcsoci/hcsdoc_wcow.go b/internal/hcsoci/hcsdoc_wcow.go index 54c4be1eaf..8f51420540 100644 --- a/internal/hcsoci/hcsdoc_wcow.go +++ b/internal/hcsoci/hcsdoc_wcow.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/uvmfolder" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -113,17 +114,17 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount // exclusive. func ConvertCPULimits(ctx context.Context, cid string, spec *specs.Spec, maxCPUCount int32) (int32, int32, int32, error) { cpuNumSet := 0 - cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, oci.AnnotationContainerProcessorCount, 0) + cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, annotations.ContainerProcessorCount, 0) if cpuCount > 0 { cpuNumSet++ } - cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, oci.AnnotationContainerProcessorLimit, 0) + cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, annotations.ContainerProcessorLimit, 0) if cpuLimit > 0 { cpuNumSet++ } - cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, oci.AnnotationContainerProcessorWeight, 0) + cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, annotations.ContainerProcessorWeight, 0) if cpuWeight > 0 { cpuNumSet++ } @@ -244,7 +245,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Memory Resources - memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, oci.AnnotationContainerMemorySizeInMB, 0) + memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, annotations.ContainerMemorySizeInMB, 0) if memoryMaxInMB > 0 { v1.MemoryMaximumInMB = int64(memoryMaxInMB) v2Container.Memory = &hcsschema.Memory{ @@ -253,8 +254,8 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Storage Resources - storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, oci.AnnotationContainerStorageQoSBandwidthMaximum, 0) - storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, oci.AnnotationContainerStorageQoSIopsMaximum, 0) + storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, annotations.ContainerStorageQoSBandwidthMaximum, 0) + storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, annotations.ContainerStorageQoSIopsMaximum, 0) if storageBandwidthMax > 0 || storageIopsMax > 0 { v1.StorageBandwidthMaximum = uint64(storageBandwidthMax) v1.StorageIOPSMaximum = uint64(storageIopsMax) @@ -392,7 +393,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter dumpPath = coi.HostingSystem.ProcessDumpLocation() } - if specDumpPath, ok := coi.Spec.Annotations[oci.AnnotationContainerProcessDumpLocation]; ok { + if specDumpPath, ok := coi.Spec.Annotations[annotations.ContainerProcessDumpLocation]; ok { // If a process dump path was specified at pod creation time for a hypervisor isolated pod, then // use this value. If one was specified on the container creation document then override with this // instead. Unlike Linux, Windows containers can set the dump path on a per container basis. @@ -466,8 +467,8 @@ func parseAssignedDevices(ctx context.Context, coi *createOptionsInternal, v2 *h // corresponding value the registry expects to be set. // // See DumpType at https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps for the mappings -func parseDumpType(annotations map[string]string) (int32, error) { - dmpTypeStr := annotations[oci.AnnotationWCOWProcessDumpType] +func parseDumpType(annots map[string]string) (int32, error) { + dmpTypeStr := annots[annotations.WCOWProcessDumpType] switch dmpTypeStr { case "": // If no type specified, default to full dumps. diff --git a/internal/hcsoci/resources_lcow.go b/internal/hcsoci/resources_lcow.go index 1bdf795ae1..c4fec89f27 100644 --- a/internal/hcsoci/resources_lcow.go +++ b/internal/hcsoci/resources_lcow.go @@ -14,9 +14,9 @@ import ( "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -25,7 +25,7 @@ import ( // shim option is set. Right now we only support Nvidia gpus, so this will default to // a gpu vhd with nvidia files func getGPUVHDPath(coi *createOptionsInternal) (string, error) { - gpuVHDPath, ok := coi.Spec.Annotations[oci.AnnotationGPUVHDPath] + gpuVHDPath, ok := coi.Spec.Annotations[annotations.GPUVHDPath] if !ok || gpuVHDPath == "" { return "", fmt.Errorf("no gpu vhd specified %s", gpuVHDPath) } diff --git a/internal/jobcontainers/oci.go b/internal/jobcontainers/oci.go index 95dfe68a40..adb5815ca3 100644 --- a/internal/jobcontainers/oci.go +++ b/internal/jobcontainers/oci.go @@ -4,9 +4,9 @@ import ( "context" "github.com/Microsoft/hcsshim/internal/hcsoci" - "github.com/Microsoft/hcsshim/internal/processorinfo" - "github.com/Microsoft/hcsshim/internal/jobobject" + "github.com/Microsoft/hcsshim/internal/processorinfo" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/internal/oci" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -16,8 +16,8 @@ const processorWeightMax = 10000 // This file contains helpers for converting parts of the oci spec to useful // structures/limits to be applied to a job object. -func getUserTokenInheritAnnotation(annotations map[string]string) bool { - val, ok := annotations[oci.AnnotationHostProcessInheritUser] +func getUserTokenInheritAnnotation(annots map[string]string) bool { + val, ok := annots[annotations.HostProcessInheritUser] return ok && val == "true" } @@ -43,11 +43,11 @@ func specToLimits(ctx context.Context, cid string, s *specs.Spec) (*jobobject.Jo } // Memory limit - memLimitMB := oci.ParseAnnotationsMemory(ctx, s, oci.AnnotationContainerMemorySizeInMB, 0) + memLimitMB := oci.ParseAnnotationsMemory(ctx, s, annotations.ContainerMemorySizeInMB, 0) // IO limits - maxBandwidth := int64(oci.ParseAnnotationsStorageBps(ctx, s, oci.AnnotationContainerStorageQoSBandwidthMaximum, 0)) - maxIops := int64(oci.ParseAnnotationsStorageIops(ctx, s, oci.AnnotationContainerStorageQoSIopsMaximum, 0)) + maxBandwidth := int64(oci.ParseAnnotationsStorageBps(ctx, s, annotations.ContainerStorageQoSBandwidthMaximum, 0)) + maxIops := int64(oci.ParseAnnotationsStorageIops(ctx, s, annotations.ContainerStorageQoSIopsMaximum, 0)) return &jobobject.JobLimits{ CPULimit: realCPULimit, diff --git a/internal/oci/annotations.go b/internal/oci/annotations.go deleted file mode 100644 index bed5230470..0000000000 --- a/internal/oci/annotations.go +++ /dev/null @@ -1,237 +0,0 @@ -package oci - -const ( - // AnnotationContainerMemorySizeInMB overrides the container memory size set - // via the OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Memory.Limit`. - AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" - - // AnnotationContainerProcessorCount overrides the container processor count - // set via the OCI spec. - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. - AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" - - // AnnotationContainerProcessorLimit overrides the container processor limit - // set via the OCI spec. - // - // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.CPU.Maximum`. - AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" - - // AnnotationContainerProcessorWeight overrides the container processor - // weight set via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. - AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" - - // AnnotationContainerStorageQoSBandwidthMaximum overrides the container - // storage bandwidth per second set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Bps`. - AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" - - // AnnotationContainerStorageQoSIopsMaximum overrides the container storage - // maximum iops set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Iops`. - AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" - - // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd - AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" - - // AnnotationVirtualMachineKernelDrivers indicates what drivers to install in the pod. - // This value should contain a list of comma separated directories containing all - // files and information needed to install given driver(s). For windows, this may - // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. - // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. - AnnotationVirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" - - // AnnotationDeviceExtensions contains a comma separated list of full paths to device extension files. - // The content of these are added to a container's hcs create document. - AnnotationDeviceExtensions = "io.microsoft.container.wcow.deviceextensions" - - // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process - // container as and instead inherit the user token from the executable that is launching the container process. - AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" - - // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). - AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" - - // AnnotationAllowOvercommit indicates if we should allow over commit memory for UVM. - // Defaults to true. For physical backed memory, set to false. - AnnotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" - - // AnnotationEnableDeferredCommit indicates if we should allow deferred memory commit for UVM. - // Defaults to false. For virtual memory with deferred commit, set to true. - AnnotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" - - // AnnotationEnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM - // to trim non-zeroed pages from the working set (if supported by the guest operating system). - AnnotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" - - // AnnotationMemorySizeInMB overrides the container memory size set via the - // OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - AnnotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" - - // AnnotationMemoryLowMMIOGapInMB indicates the low MMIO gap in MB - AnnotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO base in MB - AnnotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO gap in MB - AnnotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" - - // annotationProcessorCount overrides the hypervisor isolated vCPU count set - // via the OCI spec. - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" - - // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set - // via the OCI spec. - // - // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" - - // AnnotationProcessorWeight overrides the hypervisor isolated vCPU weight set - // via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" - - // AnnotationVPMemCount indicates the max number of vpmem devices that can be used on the UVM - AnnotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" - - // AnnotationVPMemSize indicates the size of the VPMem devices. - AnnotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" - - // AnnotationPreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. - // valid values are "initrd" or "vhd" - AnnotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" - - // AnnotationBootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM - AnnotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" - - // AnnotationKernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` - AnnotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" - - // AnnotationVPCIEnabled indicates that pci support should be enabled for the LCOW UVM - AnnotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" - - // AnnotationVPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping - AnnotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" - - // AnnotationKernelBootOptions is used to specify kernel options used while booting a linux kernel - AnnotationKernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" - - // AnnotationStorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` - // will default to the platform default. - AnnotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" - - // AnnotationStorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will - // default to the platform default. - AnnotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" - - // AnnotationFullyPhysicallyBacked indicates that the UVM should use physically backed memory only, - // including for additional devices added later. - AnnotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" - - // AnnotationDisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM - // for WCOW. - AnnotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" - - // AnnotationVSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM - AnnotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" - - // AnnotationCPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any - AnnotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" - - // AnnotationSaveAsTemplate annotation must be used with a pod & container creation request. - // If this annotation is present in the request then it will save the UVM (pod) - // and the container(s) inside it as a template. However, this also means that this - // pod and the containers inside this pod will permananetly stay in the - // paused/templated state and can not be resumed again. - AnnotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" - - // AnnotationTemplateID should be used when creating a pod or a container from a template. - // When creating a pod from a template use the ID of the templated pod as the - // TemplateID and when creating a container use the ID of the templated container as - // the TemplateID. It is the client's responsibility to make sure that the sandbox - // within which a cloned container needs to be created must also be created from the - // same template. - AnnotationTemplateID = "io.microsoft.virtualmachine.templateid" - - // AnnotationNetworkConfigProxy holds the address of the network config proxy service. - // If set, network setup will be attempted via ncproxy. - AnnotationNetworkConfigProxy = "io.microsoft.network.ncproxy" - - // AnnotationNcproxyContainerID indicates whether or not to use the hcsshim container ID - // when setting up ncproxy and computeagent - AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" - - // AnnotationEncryptedScratchDisk indicates whether or not the container scratch disks - // should be encrypted or not - AnnotationEncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" - - // AnnotationSecurityPolicy is used to specify a security policy for opengcs to enforce - AnnotationSecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" - // AnnotationContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As - // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of - // some kind (vhd, bind mount, fileshare mount etc.) - AnnotationContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" - - // AnnotationWCOWProcessDumpType specifies the type of dump to create when generating a local user mode - // process dump for Windows containers. The supported options are "mini", and "full". - // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps - AnnotationWCOWProcessDumpType = "io.microsoft.wcow.processdumptype" - - // AnnotationRLimitCore specifies the core rlimit value for a container. This will need to be set - // in order to have core dumps generated for a given container. - AnnotationRLimitCore = "io.microsoft.lcow.rlimitcore" -) diff --git a/internal/oci/sandbox.go b/internal/oci/sandbox.go index 3c3f89280d..569b035654 100644 --- a/internal/oci/sandbox.go +++ b/internal/oci/sandbox.go @@ -2,28 +2,22 @@ package oci import ( "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" ) -// KubernetesContainerTypeAnnotation is the annotation used by CRI to define the `ContainerType`. -const KubernetesContainerTypeAnnotation = "io.kubernetes.cri.container-type" - -// KubernetesSandboxIDAnnotation is the annotation used by CRI to define the -// KubernetesContainerTypeAnnotation == "sandbox"` ID. -const KubernetesSandboxIDAnnotation = "io.kubernetes.cri.sandbox-id" - // KubernetesContainerType defines the valid types of the -// `KubernetesContainerTypeAnnotation` annotation. +// `annotations.KubernetesContainerType` annotation. type KubernetesContainerType string const ( // KubernetesContainerTypeNone is only valid when - // `KubernetesContainerTypeAnnotation` is not set. + // `annotations.KubernetesContainerType` is not set. KubernetesContainerTypeNone KubernetesContainerType = "" // KubernetesContainerTypeContainer is valid when - // `KubernetesContainerTypeAnnotation == "container"`. + // `annotations.KubernetesContainerType == "container"`. KubernetesContainerTypeContainer KubernetesContainerType = "container" // KubernetesContainerTypeSandbox is valid when - // `KubernetesContainerTypeAnnotation == "sandbox"`. + // `annotations.KubernetesContainerType == "sandbox"`. KubernetesContainerTypeSandbox KubernetesContainerType = "sandbox" ) @@ -32,27 +26,27 @@ const ( // annotations and if found validates the set before returning. func GetSandboxTypeAndID(specAnnotations map[string]string) (KubernetesContainerType, string, error) { var ct KubernetesContainerType - if t, ok := specAnnotations[KubernetesContainerTypeAnnotation]; ok { + if t, ok := specAnnotations[annotations.KubernetesContainerType]; ok { switch t { case string(KubernetesContainerTypeContainer): ct = KubernetesContainerTypeContainer case string(KubernetesContainerTypeSandbox): ct = KubernetesContainerTypeSandbox default: - return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", KubernetesContainerTypeAnnotation, t) + return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", annotations.KubernetesContainerType, t) } } - id := specAnnotations[KubernetesSandboxIDAnnotation] + id := specAnnotations[annotations.KubernetesSandboxID] switch ct { case KubernetesContainerTypeContainer, KubernetesContainerTypeSandbox: if id == "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesContainerTypeAnnotation, KubernetesSandboxIDAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesContainerType, annotations.KubernetesSandboxID) } default: if id != "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesSandboxIDAnnotation, KubernetesContainerTypeAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesSandboxID, annotations.KubernetesContainerType) } } return ct, id, nil diff --git a/internal/oci/sandbox_test.go b/internal/oci/sandbox_test.go index b88765d84e..c6d1ebb8f6 100644 --- a/internal/oci/sandbox_test.go +++ b/internal/oci/sandbox_test.go @@ -1,10 +1,14 @@ package oci -import "testing" +import ( + "testing" + + "github.com/Microsoft/hcsshim/pkg/annotations" +) func Test_GetSandboxTypeAndID_TypeContainer_NoID_Failure(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "container", + annotations.KubernetesContainerType: "container", } ct, id, err := GetSandboxTypeAndID(a) if err == nil { @@ -20,7 +24,7 @@ func Test_GetSandboxTypeAndID_TypeContainer_NoID_Failure(t *testing.T) { func Test_GetSandboxTypeAndID_TypeSandbox_NoID_Failure(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "sandbox", + annotations.KubernetesContainerType: "sandbox", } ct, id, err := GetSandboxTypeAndID(a) if err == nil { @@ -36,7 +40,7 @@ func Test_GetSandboxTypeAndID_TypeSandbox_NoID_Failure(t *testing.T) { func Test_GetSandboxTypeAndID_NoType_ValidID_Failure(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.sandbox-id": t.Name(), + annotations.KubernetesSandboxID: t.Name(), } ct, id, err := GetSandboxTypeAndID(a) if err == nil { @@ -65,8 +69,8 @@ func Test_GetSandboxTypeAndID_NoAnnotations_Success(t *testing.T) { func Test_GetSandboxTypeAndID_TypeContainer_ValidID_Success(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "container", - "io.kubernetes.cri.sandbox-id": t.Name(), + annotations.KubernetesContainerType: "container", + annotations.KubernetesSandboxID: t.Name(), } ct, id, err := GetSandboxTypeAndID(a) if err != nil { @@ -82,8 +86,8 @@ func Test_GetSandboxTypeAndID_TypeContainer_ValidID_Success(t *testing.T) { func Test_GetSandboxTypeAndID_TypeSandbox_ValidID_Success(t *testing.T) { a := map[string]string{ - "io.kubernetes.cri.container-type": "sandbox", - "io.kubernetes.cri.sandbox-id": t.Name(), + annotations.KubernetesContainerType: "sandbox", + annotations.KubernetesSandboxID: t.Name(), } ct, id, err := GetSandboxTypeAndID(a) if err != nil { diff --git a/internal/oci/util.go b/internal/oci/util.go index 85f52016ff..6452241eca 100644 --- a/internal/oci/util.go +++ b/internal/oci/util.go @@ -1,6 +1,9 @@ package oci -import "github.com/opencontainers/runtime-spec/specs-go" +import ( + "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/opencontainers/runtime-spec/specs-go" +) // IsLCOW checks if `s` is a LCOW config. func IsLCOW(s *specs.Spec) bool { @@ -19,5 +22,5 @@ func IsIsolated(s *specs.Spec) bool { // IsJobContainer checks if `s` is asking for a Windows job container. func IsJobContainer(s *specs.Spec) bool { - return s.Annotations[AnnotationHostProcessContainer] == "true" + return s.Annotations[annotations.HostProcessContainer] == "true" } diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index c77a90731a..fcf41fba08 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "strings" @@ -219,13 +220,13 @@ func parseAnnotationsString(a map[string]string, key string, def string) string // if this create request should be considered as a template creation request. If value // is found the returns the actual value, returns false otherwise. func ParseAnnotationsSaveAsTemplate(ctx context.Context, s *specs.Spec) bool { - return parseAnnotationsBool(ctx, s.Annotations, AnnotationSaveAsTemplate, false) + return parseAnnotationsBool(ctx, s.Annotations, annotations.SaveAsTemplate, false) } // ParseAnnotationsTemplateID searches for the templateID in the create request. If the // value is found then returns the value otherwise returns the empty string. func ParseAnnotationsTemplateID(ctx context.Context, s *specs.Spec) string { - return parseAnnotationsString(s.Annotations, AnnotationTemplateID, "") + return parseAnnotationsString(s.Annotations, annotations.TemplateID, "") } func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, templateID string, err error) { @@ -244,7 +245,7 @@ func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, // handleAnnotationKernelDirectBoot handles parsing annotationKernelDirectBoot and setting // implied annotations from the result. func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.KernelDirect = parseAnnotationsBool(ctx, a, AnnotationKernelDirectBoot, lopts.KernelDirect) + lopts.KernelDirect = parseAnnotationsBool(ctx, a, annotations.KernelDirectBoot, lopts.KernelDirect) if !lopts.KernelDirect { lopts.KernelFile = uvm.KernelFile } @@ -253,7 +254,7 @@ func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, // handleAnnotationPreferredRootFSType handles parsing annotationPreferredRootFSType and setting // implied annotations from the result func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, AnnotationPreferredRootFSType, lopts.PreferredRootFSType) + lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, annotations.PreferredRootFSType, lopts.PreferredRootFSType) switch lopts.PreferredRootFSType { case uvm.PreferredRootFSTypeInitRd: lopts.RootFSFile = uvm.InitrdFile @@ -267,7 +268,7 @@ func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]strin func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]string, opts interface{}) { switch options := opts.(type) { case *uvm.OptionsLCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false options.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd @@ -275,7 +276,7 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str options.VPMemDeviceCount = 0 } case *uvm.OptionsWCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false } @@ -285,8 +286,8 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str // handleCloneAnnotations handles parsing annotations related to template creation and cloning // Since late cloning is only supported for WCOW this function only deals with WCOW options. func handleCloneAnnotations(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) (err error) { - wopts.IsTemplate = parseAnnotationsBool(ctx, a, AnnotationSaveAsTemplate, false) - templateID := parseAnnotationsString(a, AnnotationTemplateID, "") + wopts.IsTemplate = parseAnnotationsBool(ctx, a, annotations.SaveAsTemplate, false) + templateID := parseAnnotationsString(a, annotations.TemplateID, "") if templateID != "" { tc, err := clone.FetchTemplateConfig(ctx, templateID) if err != nil { @@ -310,29 +311,29 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( } if IsLCOW(s) { lopts := uvm.NewDefaultOptionsLCOW(id, owner) - lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, lopts.MemorySizeInMB) - lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) - lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) - lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) - lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, lopts.AllowOvercommit) - lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, lopts.EnableDeferredCommit) - lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) - lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, lopts.ProcessorCount) - lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, lopts.ProcessorLimit) - lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, lopts.ProcessorWeight) - lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, AnnotationVPMemCount, lopts.VPMemDeviceCount) - lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, AnnotationVPMemSize, lopts.VPMemSizeBytes) - lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) - lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) - lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) - lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPCIEnabled, lopts.VPCIEnabled) - lopts.BootFilesPath = parseAnnotationsString(s.Annotations, AnnotationBootFilesRootPath, lopts.BootFilesPath) - lopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, lopts.CPUGroupID) - lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, lopts.NetworkConfigProxy) - lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, AnnotationEncryptedScratchDisk, lopts.EnableScratchEncryption) - lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, AnnotationSecurityPolicy, lopts.SecurityPolicy) - lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, AnnotationKernelBootOptions, lopts.KernelBootOptions) - lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, lopts.ProcessDumpLocation) + lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, lopts.MemorySizeInMB) + lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) + lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) + lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) + lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, lopts.AllowOvercommit) + lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, lopts.EnableDeferredCommit) + lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableColdDiscardHint, lopts.EnableColdDiscardHint) + lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, lopts.ProcessorCount) + lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, lopts.ProcessorLimit) + lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, lopts.ProcessorWeight) + lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, annotations.VPMemCount, lopts.VPMemDeviceCount) + lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, annotations.VPMemSize, lopts.VPMemSizeBytes) + lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, annotations.VPMemNoMultiMapping, lopts.VPMemNoMultiMapping) + lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) + lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) + lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotations.VPCIEnabled, lopts.VPCIEnabled) + lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotations.BootFilesRootPath, lopts.BootFilesPath) + lopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, lopts.CPUGroupID) + lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, lopts.NetworkConfigProxy) + lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, annotations.EncryptedScratchDisk, lopts.EnableScratchEncryption) + lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, annotations.SecurityPolicy, lopts.SecurityPolicy) + lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) + lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, lopts.ProcessDumpLocation) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) @@ -342,22 +343,22 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( return lopts, nil } else if IsWCOW(s) { wopts := uvm.NewDefaultOptionsWCOW(id, owner) - wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, wopts.MemorySizeInMB) - wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) - wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) - wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) - wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, wopts.AllowOvercommit) - wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, wopts.EnableDeferredCommit) - wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, wopts.ProcessorCount) - wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, wopts.ProcessorLimit) - wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, wopts.ProcessorWeight) - wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) - wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) - wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, AnnotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) - wopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, wopts.CPUGroupID) - wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, wopts.NetworkConfigProxy) - wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, AnnotationVSMBNoDirectMap, wopts.NoDirectMap) - wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, wopts.ProcessDumpLocation) + wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, wopts.MemorySizeInMB) + wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) + wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) + wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) + wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, wopts.AllowOvercommit) + wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, wopts.EnableDeferredCommit) + wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, wopts.ProcessorCount) + wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, wopts.ProcessorLimit) + wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, wopts.ProcessorWeight) + wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) + wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) + wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, annotations.DisableCompartmentNamespace, wopts.DisableCompartmentNamespace) + wopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, wopts.CPUGroupID) + wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, wopts.NetworkConfigProxy) + wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, annotations.VSMBNoDirectMap, wopts.NoDirectMap) + wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, wopts.ProcessDumpLocation) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) if err := handleCloneAnnotations(ctx, s.Annotations, wopts); err != nil { return nil, err @@ -374,24 +375,24 @@ func UpdateSpecFromOptions(s specs.Spec, opts *runhcsopts.Options) specs.Spec { return s } - if _, ok := s.Annotations[AnnotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { - s.Annotations[AnnotationBootFilesRootPath] = opts.BootFilesRootPath + if _, ok := s.Annotations[annotations.BootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { + s.Annotations[annotations.BootFilesRootPath] = opts.BootFilesRootPath } - if _, ok := s.Annotations[AnnotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { - s.Annotations[AnnotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) + if _, ok := s.Annotations[annotations.ProcessorCount]; !ok && opts.VmProcessorCount != 0 { + s.Annotations[annotations.ProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) } - if _, ok := s.Annotations[AnnotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { - s.Annotations[AnnotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) + if _, ok := s.Annotations[annotations.MemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { + s.Annotations[annotations.MemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) } - if _, ok := s.Annotations[AnnotationGPUVHDPath]; !ok && opts.GPUVHDPath != "" { - s.Annotations[AnnotationGPUVHDPath] = opts.GPUVHDPath + if _, ok := s.Annotations[annotations.GPUVHDPath]; !ok && opts.GPUVHDPath != "" { + s.Annotations[annotations.GPUVHDPath] = opts.GPUVHDPath } - if _, ok := s.Annotations[AnnotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { - s.Annotations[AnnotationNetworkConfigProxy] = opts.NCProxyAddr + if _, ok := s.Annotations[annotations.NetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { + s.Annotations[annotations.NetworkConfigProxy] = opts.NCProxyAddr } return s diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 14b1691951..8653d9142b 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -4,6 +4,7 @@ import ( "testing" runhcsopts "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -15,12 +16,12 @@ func Test_SpecUpdate_MemorySize_WithAnnotation_WithOpts(t *testing.T) { s := &specs.Spec{ Linux: &specs.Linux{}, Annotations: map[string]string{ - AnnotationMemorySizeInMB: "2048", + annotations.MemorySizeInMB: "2048", }, } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationMemorySizeInMB] != "2048" { + if updatedSpec.Annotations[annotations.MemorySizeInMB] != "2048" { t.Fatal("should not have updated annotation to default when annotation is provided in the spec") } } @@ -36,7 +37,7 @@ func Test_SpecUpdate_MemorySize_NoAnnotation_WithOpts(t *testing.T) { } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationMemorySizeInMB] != "3072" { + if updatedSpec.Annotations[annotations.MemorySizeInMB] != "3072" { t.Fatal("should have updated annotation to default when annotation is not provided in the spec") } } @@ -49,12 +50,12 @@ func Test_SpecUpdate_ProcessorCount_WithAnnotation_WithOpts(t *testing.T) { s := &specs.Spec{ Linux: &specs.Linux{}, Annotations: map[string]string{ - AnnotationProcessorCount: "8", + annotations.ProcessorCount: "8", }, } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationProcessorCount] != "8" { + if updatedSpec.Annotations[annotations.ProcessorCount] != "8" { t.Fatal("should not have updated annotation to default when annotation is provided in the spec") } } @@ -70,7 +71,7 @@ func Test_SpecUpdate_ProcessorCount_NoAnnotation_WithOpts(t *testing.T) { } updatedSpec := UpdateSpecFromOptions(*s, opts) - if updatedSpec.Annotations[AnnotationProcessorCount] != "4" { + if updatedSpec.Annotations[annotations.ProcessorCount] != "4" { t.Fatal("should have updated annotation to default when annotation is not provided in the spec") } } diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go new file mode 100644 index 0000000000..0042b3d24c --- /dev/null +++ b/pkg/annotations/annotations.go @@ -0,0 +1,253 @@ +package annotations + +const ( + // ContainerMemorySizeInMB overrides the container memory size set + // via the OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Memory.Limit`. + ContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" + + // ContainerProcessorCount overrides the container processor count + // set via the OCI spec. + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. + ContainerProcessorCount = "io.microsoft.container.processor.count" + + // ContainerProcessorLimit overrides the container processor limit + // set via the OCI spec. + // + // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.CPU.Maximum`. + ContainerProcessorLimit = "io.microsoft.container.processor.limit" + + // ContainerProcessorWeight overrides the container processor + // weight set via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. + ContainerProcessorWeight = "io.microsoft.container.processor.weight" + + // ContainerStorageQoSBandwidthMaximum overrides the container + // storage bandwidth per second set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Bps`. + ContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" + + // ContainerStorageQoSIopsMaximum overrides the container storage + // maximum iops set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Iops`. + ContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" + + // GPUVHDPath overrides the default path to search for the gpu vhd + GPUVHDPath = "io.microsoft.lcow.gpuvhdpath" + + // ContainerGPUCapabilities is used to find the gpu capabilities on the container spec + ContainerGPUCapabilities = "io.microsoft.container.gpu.capabilities" + + // VirtualMachineKernelDrivers indicates what drivers to install in the pod. + // This value should contain a list of comma separated directories containing all + // files and information needed to install given driver(s). For windows, this may + // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. + // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. + VirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" + + // DeviceExtensions contains a comma separated list of full paths to device extension files. + // The content of these are added to a container's hcs create document. + DeviceExtensions = "io.microsoft.container.wcow.deviceextensions" + + // HostProcessInheritUser indicates whether to ignore the username passed in to run a host process + // container as and instead inherit the user token from the executable that is launching the container process. + HostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" + + // HostProcessContainer indicates to launch a host process container (job container in this repository). + HostProcessContainer = "microsoft.com/hostprocess-container" + + // AllowOvercommit indicates if we should allow over commit memory for UVM. + // Defaults to true. For physical backed memory, set to false. + AllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" + + // EnableDeferredCommit indicates if we should allow deferred memory commit for UVM. + // Defaults to false. For virtual memory with deferred commit, set to true. + EnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" + + // EnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM + // to trim non-zeroed pages from the working set (if supported by the guest operating system). + EnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" + + // MemorySizeInMB overrides the container memory size set via the + // OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + MemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" + + // MemoryLowMMIOGapInMB indicates the low MMIO gap in MB + MemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO base in MB + MemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO gap in MB + MemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" + + // ProcessorCount overrides the hypervisor isolated vCPU count set + // via the OCI spec. + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" + + // ProcessorLimit overrides the hypervisor isolated vCPU limit set + // via the OCI spec. + // + // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" + + // ProcessorWeight overrides the hypervisor isolated vCPU weight set + // via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" + + // VPMemCount indicates the max number of vpmem devices that can be used on the UVM + VPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" + + // VPMemSize indicates the size of the VPMem devices. + VPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" + + // PreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. + // valid values are "initrd" or "vhd" + PreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" + + // BootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM + BootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" + + // KernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` + KernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" + + // VPCIEnabled indicates that pci support should be enabled for the LCOW UVM + VPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" + + // VPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping + VPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" + + // KernelBootOptions is used to specify kernel options used while booting a linux kernel + KernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" + + // StorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` + // will default to the platform default. + StorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" + + // StorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will + // default to the platform default. + StorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" + + // FullyPhysicallyBacked indicates that the UVM should use physically backed memory only, + // including for additional devices added later. + FullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" + + // DisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM + // for WCOW. + DisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" + + // VSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM + VSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" + + // CPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any + CPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" + + // SaveAsTemplate annotation must be used with a pod & container creation request. + // If this annotation is present in the request then it will save the UVM (pod) + // and the container(s) inside it as a template. However, this also means that this + // pod and the containers inside this pod will permananetly stay in the + // paused/templated state and can not be resumed again. + SaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" + + // TemplateID should be used when creating a pod or a container from a template. + // When creating a pod from a template use the ID of the templated pod as the + // TemplateID and when creating a container use the ID of the templated container as + // the TemplateID. It is the client's responsibility to make sure that the sandbox + // within which a cloned container needs to be created must also be created from the + // same template. + TemplateID = "io.microsoft.virtualmachine.templateid" + + // NetworkConfigProxy holds the address of the network config proxy service. + // If set, network setup will be attempted via ncproxy. + NetworkConfigProxy = "io.microsoft.network.ncproxy" + + // NcproxyContainerID indicates whether or not to use the hcsshim container ID + // when setting up ncproxy and computeagent + NcproxyContainerID = "io.microsoft.network.ncproxy.containerid" + + // EncryptedScratchDisk indicates whether or not the container scratch disks + // should be encrypted or not + EncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + + // SecurityPolicy is used to specify a security policy for opengcs to enforce + SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // ContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As + // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of + // some kind (vhd, bind mount, fileshare mount etc.) + ContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" + + // WCOWProcessDumpType specifies the type of dump to create when generating a local user mode + // process dump for Windows containers. The supported options are "mini", and "full". + // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps + WCOWProcessDumpType = "io.microsoft.wcow.processdumptype" + + // RLimitCore specifies the core rlimit value for a container. This will need to be set + // in order to have core dumps generated for a given container. + RLimitCore = "io.microsoft.lcow.rlimitcore" + + // LCOWDevShmSizeInKb specifies the size of LCOW /dev/shm. + LCOWDevShmSizeInKb = "io.microsoft.lcow.shm.size-kb" + + // LCOWPrivileged is used to specify that the container should be run in privileged mode + LCOWPrivileged = "io.microsoft.virtualmachine.lcow.privileged" + + // KubernetesContainerType is the annotation used by CRI to define the `ContainerType`. + KubernetesContainerType = "io.kubernetes.cri.container-type" + + // KubernetesSandboxID is the annotation used by CRI to define the + // KubernetesContainerType == "sandbox"` ID. + KubernetesSandboxID = "io.kubernetes.cri.sandbox-id" +) diff --git a/test/containerd-shim-runhcs-v1/start_test.go b/test/containerd-shim-runhcs-v1/start_test.go index 4ff5dc7cdc..91249956c4 100644 --- a/test/containerd-shim-runhcs-v1/start_test.go +++ b/test/containerd-shim-runhcs-v1/start_test.go @@ -15,6 +15,7 @@ import ( "testing" "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/containerd/containerd/runtime/v2/task" "github.com/containerd/ttrpc" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -164,8 +165,8 @@ func Test_Start_Pod_Config(t *testing.T) { t.Fatalf("failed to generate Windows config with error: %v", err) } // Setup the POD annotations - g.AddAnnotation("io.kubernetes.cri.container-type", "sandbox") - g.AddAnnotation("io.kubernetes.cri.sandbox-id", t.Name()) + g.AddAnnotation(annotations.KubernetesContainerType, "sandbox") + g.AddAnnotation(annotations.KubernetesSandboxID, t.Name()) writeBundleConfig(t, cmd.Dir, g.Config) @@ -183,8 +184,8 @@ func Test_Start_Container_InPod_Config(t *testing.T) { t.Fatalf("failed to generate Windows config with error: %v", perr) } - pg.AddAnnotation("io.kubernetes.cri.container-type", "sandbox") - pg.AddAnnotation("io.kubernetes.cri.sandbox-id", podID) + pg.AddAnnotation(annotations.KubernetesContainerType, "sandbox") + pg.AddAnnotation(annotations.KubernetesSandboxID, podID) writeBundleConfig(t, pcmd.Dir, pg.Config) @@ -203,8 +204,8 @@ func Test_Start_Container_InPod_Config(t *testing.T) { } // Setup the POD Workload container annotations - wg.AddAnnotation("io.kubernetes.cri.container-type", "container") - wg.AddAnnotation("io.kubernetes.cri.sandbox-id", podID) + wg.AddAnnotation(annotations.KubernetesContainerType, "container") + wg.AddAnnotation(annotations.KubernetesSandboxID, podID) writeBundleConfig(t, wcmd.Dir, wg.Config) @@ -222,7 +223,7 @@ func Test_Start_Container_InPod_Config_PodShim_Gone(t *testing.T) { podID := "POD-TEST" // Setup the POD Workload container annotations - g.AddAnnotation("io.kubernetes.cri.container-type", "container") + g.AddAnnotation(annotations.KubernetesContainerType, "container") g.AddAnnotation("io.kubernetes.cri.sandbox-id", podID) writeBundleConfig(t, cmd.Dir, g.Config) diff --git a/test/cri-containerd/clone_test.go b/test/cri-containerd/clone_test.go index a816e29352..01e675f846 100644 --- a/test/cri-containerd/clone_test.go +++ b/test/cri-containerd/clone_test.go @@ -11,8 +11,8 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -27,7 +27,7 @@ func getTemplatePodConfig(name string) *runtime.RunPodSandboxRequest { Namespace: testNamespace, }, Annotations: map[string]string{ - oci.AnnotationSaveAsTemplate: "true", + annotations.SaveAsTemplate: "true", }, }, RuntimeHandler: wcowHypervisorRuntimeHandler, @@ -50,7 +50,7 @@ func getTemplateContainerConfig(name string) *runtime.CreateContainerRequest { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationSaveAsTemplate: "true", + annotations.SaveAsTemplate: "true", }, }, } @@ -85,7 +85,7 @@ func getClonedPodConfig(uniqueID int, templateid string) *runtime.RunPodSandboxR Namespace: testNamespace, }, Annotations: map[string]string{ - oci.AnnotationTemplateID: templateid + "@vm", + annotations.TemplateID: templateid + "@vm", }, }, RuntimeHandler: wcowHypervisorRuntimeHandler, @@ -109,7 +109,7 @@ func getClonedContainerConfig(uniqueID int, templateid string) *runtime.CreateCo "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationTemplateID: templateid, + annotations.TemplateID: templateid, }, }, } @@ -488,7 +488,7 @@ func Test_VerifyCloneAndTemplateConfig(t *testing.T) { // change pod config to make sure the request fails cloneSandboxRequest := getClonedPodConfig(0, templatePodID) - cloneSandboxRequest.Config.Annotations[oci.AnnotationAllowOvercommit] = "false" + cloneSandboxRequest.Config.Annotations[annotations.AllowOvercommit] = "false" _, err := client.RunPodSandbox(ctx, cloneSandboxRequest) if err == nil { diff --git a/test/cri-containerd/container_layers_packing_test.go b/test/cri-containerd/container_layers_packing_test.go index 6006229fbb..63f8a1bcf8 100644 --- a/test/cri-containerd/container_layers_packing_test.go +++ b/test/cri-containerd/container_layers_packing_test.go @@ -10,9 +10,9 @@ import ( "strings" "testing" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" ) @@ -101,10 +101,10 @@ func Test_Container_Layer_Packing_On_VPMem(t *testing.T) { }, } { t.Run(fmt.Sprintf("PreferredRootFSType-%s", scenario.rootfsType), func(t *testing.T) { - annotations := map[string]string{ - oci.AnnotationPreferredRootFSType: scenario.rootfsType, + annots := map[string]string{ + annotations.PreferredRootFSType: scenario.rootfsType, } - podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, podReq) defer removePodSandbox(t, client, ctx, podID) @@ -170,10 +170,10 @@ func Test_Annotation_Disable_Multi_Mapping(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, alpine70ExtraLayers}) - annotations := map[string]string{ - oci.AnnotationVPMemNoMultiMapping: "true", + annots := map[string]string{ + annotations.VPMemNoMultiMapping: "true", } - podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + podReq := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, podReq) defer removePodSandbox(t, client, ctx, podID) diff --git a/test/cri-containerd/container_test.go b/test/cri-containerd/container_test.go index a603c74c41..c89f4c7cc8 100644 --- a/test/cri-containerd/container_test.go +++ b/test/cri-containerd/container_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/sirupsen/logrus" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -268,8 +268,8 @@ func Test_RunContainer_ZeroVPMEM_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "initrd", - oci.AnnotationVPMemCount: "0", + annotations.PreferredRootFSType: "initrd", + annotations.VPMemCount: "0", }), ) @@ -310,8 +310,8 @@ func Test_RunContainer_ZeroVPMEM_Multiple_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "initrd", - oci.AnnotationVPMemCount: "0", + annotations.PreferredRootFSType: "initrd", + annotations.VPMemCount: "0", }), ) @@ -794,7 +794,7 @@ func Test_CreateContainer_DevShmSize(t *testing.T) { // the /dev/shm size is expected to be in KB, set it to 256 MB size := 256 * 1024 contReq1.Config.Annotations = map[string]string{ - "io.microsoft.container.storage.shm.size-kb": strconv.Itoa(size), + annotations.LCOWDevShmSizeInKb: strconv.Itoa(size), } containerID1 := createContainer(t, client, ctx, contReq1) defer removeContainer(t, client, ctx, containerID1) @@ -836,12 +836,12 @@ func Test_CreateContainer_HugePageMount_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - annotations := map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", - oci.AnnotationMemorySizeInMB: "2048", - oci.AnnotationKernelBootOptions: "hugepagesz=2M hugepages=10", + annots := map[string]string{ + annotations.FullyPhysicallyBacked: "true", + annotations.MemorySizeInMB: "2048", + annotations.KernelBootOptions: "hugepagesz=2M hugepages=10", } - sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sandboxRequest) defer removePodSandbox(t, client, ctx, podID) diff --git a/test/cri-containerd/container_update_test.go b/test/cri-containerd/container_update_test.go index 102e0745ec..0fa049d8e5 100644 --- a/test/cri-containerd/container_update_test.go +++ b/test/cri-containerd/container_update_test.go @@ -7,8 +7,8 @@ import ( "fmt" "testing" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -312,7 +312,7 @@ func Test_Container_UpdateResources_Memory(t *testing.T) { }, Command: test.cmd, Annotations: map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), // 768MB + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), // 768MB }, }, PodSandboxId: podID, diff --git a/test/cri-containerd/container_virtual_device_test.go b/test/cri-containerd/container_virtual_device_test.go index 155e433439..8a477639a2 100644 --- a/test/cri-containerd/container_virtual_device_test.go +++ b/test/cri-containerd/container_virtual_device_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -116,16 +116,16 @@ func findTestVirtualDeviceID() (string, error) { } var lcowPodGPUAnnotations = map[string]string{ - oci.AnnotationKernelDirectBoot: "false", - oci.AnnotationAllowOvercommit: "false", - oci.AnnotationPreferredRootFSType: "initrd", - oci.AnnotationVPMemCount: "0", - oci.AnnotationVPCIEnabled: "true", + annotations.KernelDirectBoot: "false", + annotations.AllowOvercommit: "false", + annotations.PreferredRootFSType: "initrd", + annotations.VPMemCount: "0", + annotations.VPCIEnabled: "true", // we believe this is a sufficiently large high MMIO space amount for this test. // if a given gpu device needs more, this test will fail to create the container // and may hang. - oci.AnnotationMemoryHighMMIOGapInMB: "64000", - oci.AnnotationBootFilesRootPath: testGPUBootFiles, + annotations.MemoryHighMMIOGapInMB: "64000", + annotations.BootFilesRootPath: testGPUBootFiles, } func getGPUContainerRequestLCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig, device *runtime.Device) *runtime.CreateContainerRequest { @@ -145,7 +145,7 @@ func getGPUContainerRequestLCOW(t *testing.T, podID string, podConfig *runtime.P }, Linux: &runtime.LinuxContainerConfig{}, Annotations: map[string]string{ - "io.microsoft.container.gpu.capabilities": "utility", + annotations.ContainerGPUCapabilities: "utility", }, }, PodSandboxId: podID, @@ -179,7 +179,7 @@ func getGPUContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P }, }, Annotations: map[string]string{ - oci.AnnotationVirtualMachineKernelDrivers: testDriversPath, + annotations.VirtualMachineKernelDrivers: testDriversPath, }, }, PodSandboxId: podID, @@ -512,7 +512,7 @@ func Test_RunContainer_VirtualDevice_GPU_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) @@ -560,7 +560,7 @@ func Test_RunContainer_VirtualDevice_GPU_and_NoGPU_WCOW_Hypervisor(t *testing.T) t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) @@ -625,7 +625,7 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_WCOW_Hypervisor(t *testing.T) t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) @@ -680,7 +680,7 @@ func Test_RunContainer_VirtualDevice_GPU_Multiple_Removal_WCOW_Hypervisor(t *tes t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) diff --git a/test/cri-containerd/createcontainer_test.go b/test/cri-containerd/createcontainer_test.go index aaaebfb804..aaff8fcd68 100644 --- a/test/cri-containerd/createcontainer_test.go +++ b/test/cri-containerd/createcontainer_test.go @@ -11,8 +11,8 @@ import ( "testing" "github.com/Microsoft/go-winio" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -305,7 +305,7 @@ func Test_CreateContainer_MemorySize_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB }, }, } @@ -365,7 +365,7 @@ func Test_CreateContainer_MemorySize_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", 768*1024*1024), // 768MB }, }, } @@ -452,7 +452,7 @@ func Test_CreateContainer_CPUCount_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorCount: "1", + annotations.ContainerProcessorCount: "1", }, }, } @@ -512,7 +512,7 @@ func Test_CreateContainer_CPUCount_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorCount: "1", + annotations.ContainerProcessorCount: "1", }, }, } @@ -599,7 +599,7 @@ func Test_CreateContainer_CPULimit_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorLimit: "9000", + annotations.ContainerProcessorLimit: "9000", }, }, } @@ -659,7 +659,7 @@ func Test_CreateContainer_CPULimit_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorLimit: "9000", + annotations.ContainerProcessorLimit: "9000", }, }, } @@ -747,7 +747,7 @@ func Test_CreateContainer_CPUWeight_Annotation_WCOW_Process(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorWeight: "500", + annotations.ContainerProcessorWeight: "500", }, }, } @@ -807,7 +807,7 @@ func Test_CreateContainer_CPUWeight_Annotation_WCOW_Hypervisor(t *testing.T) { "127.0.0.1", }, Annotations: map[string]string{ - oci.AnnotationContainerProcessorLimit: "500", + annotations.ContainerProcessorLimit: "500", }, }, } diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index 1113e4bae8..5c82248f33 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -15,7 +15,7 @@ import ( "github.com/Microsoft/go-winio/vhd" "github.com/Microsoft/hcsshim/hcn" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -24,7 +24,7 @@ func getJobContainerPodRequestWCOW(t *testing.T) *runtime.RunPodSandboxRequest { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationHostProcessContainer: "true", + annotations.HostProcessContainer: "true", }), ) } @@ -47,8 +47,8 @@ func getJobContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P }, Mounts: mounts, Annotations: map[string]string{ - oci.AnnotationHostProcessContainer: "true", - oci.AnnotationHostProcessInheritUser: "true", + annotations.HostProcessContainer: "true", + annotations.HostProcessInheritUser: "true", }, Windows: &runtime.WindowsContainerConfig{}, }, @@ -505,8 +505,8 @@ func Test_RunContainer_WorkingDirectory_JobContainer_WCOW(t *testing.T) { Command: test.cmd, WorkingDir: test.workDir, Annotations: map[string]string{ - oci.AnnotationHostProcessContainer: "true", - oci.AnnotationHostProcessInheritUser: "true", + annotations.HostProcessContainer: "true", + annotations.HostProcessInheritUser: "true", }, Windows: &runtime.WindowsContainerConfig{}, }, diff --git a/test/cri-containerd/pod_update_test.go b/test/cri-containerd/pod_update_test.go index b80efff646..a80c536c36 100644 --- a/test/cri-containerd/pod_update_test.go +++ b/test/cri-containerd/pod_update_test.go @@ -7,7 +7,7 @@ import ( "fmt" "testing" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -47,7 +47,7 @@ func Test_Pod_UpdateResources_Memory(t *testing.T) { t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), }), ) @@ -118,8 +118,8 @@ func Test_Pod_UpdateResources_Memory_PA(t *testing.T) { t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", - oci.AnnotationContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), + annotations.FullyPhysicallyBacked: "true", + annotations.ContainerMemorySizeInMB: fmt.Sprintf("%d", startingMemorySize), }), ) diff --git a/test/cri-containerd/runpodsandbox_test.go b/test/cri-containerd/runpodsandbox_test.go index 83f75ea408..28261f8425 100644 --- a/test/cri-containerd/runpodsandbox_test.go +++ b/test/cri-containerd/runpodsandbox_test.go @@ -6,6 +6,7 @@ import ( "bufio" "context" "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" "io/ioutil" "os" "path/filepath" @@ -17,7 +18,6 @@ import ( "github.com/Microsoft/hcsshim/internal/cpugroup" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/lcow" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/processorinfo" "github.com/Microsoft/hcsshim/osversion" testutilities "github.com/Microsoft/hcsshim/test/functional/utilities" @@ -121,7 +121,7 @@ func Test_RunPodSandbox_VirtualMemory_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annotations.AllowOvercommit: "true", }), ) runPodSandboxTest(t, request) @@ -136,7 +136,7 @@ func Test_RunPodSandbox_VirtualMemory_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annotations.AllowOvercommit: "true", }), ) runPodSandboxTest(t, request) @@ -151,8 +151,8 @@ func Test_RunPodSandbox_VirtualMemory_DeferredCommit_WCOW_Hypervisor(t *testing. t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", - oci.AnnotationEnableDeferredCommit: "true", + annotations.AllowOvercommit: "true", + annotations.EnableDeferredCommit: "true", }), ) runPodSandboxTest(t, request) @@ -167,8 +167,8 @@ func Test_RunPodSandbox_VirtualMemory_DeferredCommit_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "true", - oci.AnnotationEnableDeferredCommit: "true", + annotations.AllowOvercommit: "true", + annotations.EnableDeferredCommit: "true", }), ) runPodSandboxTest(t, request) @@ -183,7 +183,7 @@ func Test_RunPodSandbox_PhysicalMemory_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "false", + annotations.AllowOvercommit: "false", }), ) runPodSandboxTest(t, request) @@ -198,7 +198,7 @@ func Test_RunPodSandbox_FullyPhysicallyBacked_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) runPodSandboxTest(t, request) @@ -213,7 +213,7 @@ func Test_RunPodSandbox_VSMBNoDirectMap_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationVSMBNoDirectMap: "true", + annotations.VSMBNoDirectMap: "true", }), ) runPodSandboxTest(t, request) @@ -228,7 +228,7 @@ func Test_RunPodSandbox_PhysicalMemory_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "false", + annotations.AllowOvercommit: "false", }), ) runPodSandboxTest(t, request) @@ -243,7 +243,7 @@ func Test_RunPodSandbox_FullyPhysicallyBacked_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", + annotations.FullyPhysicallyBacked: "true", }), ) runPodSandboxTest(t, request) @@ -258,7 +258,7 @@ func Test_RunPodSandbox_MemorySize_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerMemorySizeInMB: "128", + annotations.ContainerMemorySizeInMB: "128", }), ) runPodSandboxTest(t, request) @@ -273,7 +273,7 @@ func Test_RunPodSandbox_MemorySize_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemorySizeInMB: "768", // 128 is too small for WCOW. It is really slow boot. + annotations.MemorySizeInMB: "768", // 128 is too small for WCOW. It is really slow boot. }), ) runPodSandboxTest(t, request) @@ -288,7 +288,7 @@ func Test_RunPodSandbox_MemorySize_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemorySizeInMB: "200", + annotations.MemorySizeInMB: "200", }), ) runPodSandboxTest(t, request) @@ -306,9 +306,9 @@ func Test_RunPodSandbox_MMIO_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemoryLowMMIOGapInMB: "100", - oci.AnnotationMemoryHighMMIOBaseInMB: "100", - oci.AnnotationMemoryHighMMIOGapInMB: "100", + annotations.MemoryLowMMIOGapInMB: "100", + annotations.MemoryHighMMIOBaseInMB: "100", + annotations.MemoryHighMMIOGapInMB: "100", }), ) runPodSandboxTest(t, request) @@ -326,9 +326,9 @@ func Test_RunPodSandbox_MMIO_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemoryLowMMIOGapInMB: "100", - oci.AnnotationMemoryHighMMIOBaseInMB: "100", - oci.AnnotationMemoryHighMMIOGapInMB: "100", + annotations.MemoryLowMMIOGapInMB: "100", + annotations.MemoryHighMMIOBaseInMB: "100", + annotations.MemoryHighMMIOGapInMB: "100", }), ) runPodSandboxTest(t, request) @@ -346,9 +346,9 @@ func Test_RunPodSandbox_MMIO_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationMemoryLowMMIOGapInMB: "100", - oci.AnnotationMemoryHighMMIOBaseInMB: "100", - oci.AnnotationMemoryHighMMIOGapInMB: "100", + annotations.MemoryLowMMIOGapInMB: "100", + annotations.MemoryHighMMIOBaseInMB: "100", + annotations.MemoryHighMMIOGapInMB: "100", }), ) runPodSandboxTest(t, request) @@ -363,7 +363,7 @@ func Test_RunPodSandbox_CPUCount_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorCount: "1", + annotations.ContainerProcessorCount: "1", }), ) runPodSandboxTest(t, request) @@ -378,7 +378,7 @@ func Test_RunPodSandbox_CPUCount_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorCount: "1", + annotations.ProcessorCount: "1", }), ) runPodSandboxTest(t, request) @@ -393,7 +393,7 @@ func Test_RunPodSandbox_CPUCount_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorCount: "1", + annotations.ProcessorCount: "1", }), ) runPodSandboxTest(t, request) @@ -408,7 +408,7 @@ func Test_RunPodSandbox_CPULimit_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorLimit: "9000", + annotations.ContainerProcessorLimit: "9000", }), ) runPodSandboxTest(t, request) @@ -423,7 +423,7 @@ func Test_RunPodSandbox_CPULimit_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorLimit: "90000", + annotations.ProcessorLimit: "90000", }), ) runPodSandboxTest(t, request) @@ -438,7 +438,7 @@ func Test_RunPodSandbox_CPULimit_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorLimit: "90000", + annotations.ProcessorLimit: "90000", }), ) runPodSandboxTest(t, request) @@ -453,7 +453,7 @@ func Test_RunPodSandbox_CPUWeight_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorWeight: "500", + annotations.ContainerProcessorWeight: "500", }), ) runPodSandboxTest(t, request) @@ -468,7 +468,7 @@ func Test_RunPodSandbox_CPUWeight_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessorWeight: "500", + annotations.ContainerProcessorWeight: "500", }), ) runPodSandboxTest(t, request) @@ -483,7 +483,7 @@ func Test_RunPodSandbox_CPUWeight_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationProcessorWeight: "500", + annotations.ProcessorWeight: "500", }), ) runPodSandboxTest(t, request) @@ -498,7 +498,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s + annotations.ContainerStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }), ) runPodSandboxTest(t, request) @@ -513,7 +513,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s + annotations.StorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }), ) runPodSandboxTest(t, request) @@ -528,7 +528,7 @@ func Test_RunPodSandbox_StorageQoSBandwithMax_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s + annotations.StorageQoSBandwidthMaximum: fmt.Sprintf("%d", 1024*1024), // 1MB/s }), ) runPodSandboxTest(t, request) @@ -543,7 +543,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Process(t *testing.T) { t, wcowProcessRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerStorageQoSIopsMaximum: "300", + annotations.ContainerStorageQoSIopsMaximum: "300", }), ) runPodSandboxTest(t, request) @@ -558,7 +558,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSIopsMaximum: "300", + annotations.StorageQoSIopsMaximum: "300", }), ) runPodSandboxTest(t, request) @@ -573,7 +573,7 @@ func Test_RunPodSandbox_StorageQoSIopsMax_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationStorageQoSIopsMaximum: "300", + annotations.StorageQoSIopsMaximum: "300", }), ) runPodSandboxTest(t, request) @@ -588,7 +588,7 @@ func Test_RunPodSandbox_InitrdBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "initrd", + annotations.PreferredRootFSType: "initrd", }), ) runPodSandboxTest(t, request) @@ -603,7 +603,7 @@ func Test_RunPodSandbox_RootfsVhdBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationPreferredRootFSType: "vhd", + annotations.PreferredRootFSType: "vhd", }), ) runPodSandboxTest(t, request) @@ -618,7 +618,7 @@ func Test_RunPodSandbox_VPCIEnabled_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationVPCIEnabled: "true", + annotations.VPCIEnabled: "true", }), ) runPodSandboxTest(t, request) @@ -633,7 +633,7 @@ func Test_RunPodSandbox_UEFIBoot_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationKernelDirectBoot: "false", + annotations.KernelDirectBoot: "false", }), ) runPodSandboxTest(t, request) @@ -737,11 +737,11 @@ func Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } - output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annotations) + output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annots) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW", errorMsg, exitCode) @@ -781,12 +781,12 @@ func Test_RunPodSandbox_CustomizableScratchCustomSize_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause}) - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", "containerd.io/snapshot/io.microsoft.container.storage.rootfs.size-gb": "200", } - output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annotations) + output, errorMsg, exitCode := createSandboxContainerAndExecForCustomScratch(t, annots) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, Test_RunPodSandbox_CustomizableScratchDefaultSize_LCOW", errorMsg, exitCode) @@ -828,8 +828,8 @@ func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } mounts := []*runtime.Mount{ @@ -842,7 +842,7 @@ func Test_RunPodSandbox_Mount_SandboxDir_LCOW(t *testing.T) { "mount", } - output, errorMsg, exitCode := createSandboxContainerAndExec(t, annotations, mounts, cmd) + output, errorMsg, exitCode := createSandboxContainerAndExec(t, annots, mounts, cmd) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, t.Name()) @@ -1040,7 +1040,7 @@ func Test_RunPodSandbox_CPUGroup(t *testing.T) { Namespace: testNamespace, }, Annotations: map[string]string{ - oci.AnnotationCPUGroupID: presentID, + annotations.CPUGroupID: presentID, }, }, RuntimeHandler: test.runtimeHandler, @@ -1067,8 +1067,8 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } // Create a temporary ext4 VHD to mount into the container. @@ -1089,7 +1089,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { }, } - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sbRequest) defer removePodSandbox(t, client, ctx, podID) @@ -1107,7 +1107,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { // create 2 containers with vhd mounts and verify both can mount vhd for i := 1; i < 3; i++ { containerName := t.Name() + "-Container-" + strconv.Itoa(i) - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1126,7 +1126,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_LCOW(t *testing.T) { // at the same time containers in a pod that don't have any mounts mounts = []*runtime.Mount{} containerName := t.Name() + "-Container-3" - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageLcowAlpine, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1280,8 +1280,8 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - annotations := map[string]string{ - oci.AnnotationAllowOvercommit: "true", + annots := map[string]string{ + annotations.AllowOvercommit: "true", } vhdHostDir, err := ioutil.TempDir("", "") @@ -1305,7 +1305,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { }, } - sbRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(annotations)) + sbRequest := getRunPodSandboxRequest(t, wcowHypervisorRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sbRequest) defer removePodSandbox(t, client, ctx, podID) @@ -1327,7 +1327,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { // create 2 containers with vhd mounts and verify both can mount vhd for i := 1; i < 3; i++ { containerName := t.Name() + "-Container-" + strconv.Itoa(i) - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1347,7 +1347,7 @@ func Test_RunPodSandbox_MultipleContainersSameVhd_WCOW(t *testing.T) { // at the same time containers in a pod that don't have any mounts mounts = []*runtime.Mount{} containerName := t.Name() + "-Container-3" - containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annotations, mounts, sbRequest.Config) + containerId := createContainerInSandbox(t, client, ctx, podID, containerName, imageWindowsNanoserver, command, annots, mounts, sbRequest.Config) defer removeContainer(t, client, ctx, containerId) startContainer(t, client, ctx, containerId) @@ -1374,7 +1374,7 @@ func Test_RunPodSandbox_ProcessDump_LCOW(t *testing.T) { t, lcowRuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessDumpLocation: "/coredumps/core", + annotations.ContainerProcessDumpLocation: "/coredumps/core", }), ) @@ -1389,8 +1389,8 @@ func Test_RunPodSandbox_ProcessDump_LCOW(t *testing.T) { }, } - annotations := map[string]string{ - oci.AnnotationRLimitCore: "18446744073709551615;18446744073709551615", + annots := map[string]string{ + annotations.RLimitCore: "18446744073709551615;18446744073709551615", } // Setup container 1 that uses an image that stackoverflows shortly after starting. @@ -1406,7 +1406,7 @@ func Test_RunPodSandbox_ProcessDump_LCOW(t *testing.T) { Command: []string{ "./stackoverflow", }, - Annotations: annotations, + Annotations: annots, Mounts: mounts, }, PodSandboxId: podID, @@ -1485,7 +1485,7 @@ func Test_RunPodSandbox_ProcessDump_WCOW_Hypervisor(t *testing.T) { t, wcowHypervisor19041RuntimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationContainerProcessDumpLocation: "C:\\processdump", + annotations.ContainerProcessDumpLocation: "C:\\processdump", }), ) @@ -1589,18 +1589,18 @@ func Test_RunPodSandbox_ProcessDump_WCOW_Hypervisor(t *testing.T) { } } -func createSandboxContainerAndExecForCustomScratch(t *testing.T, annotations map[string]string) (string, string, int) { +func createSandboxContainerAndExecForCustomScratch(t *testing.T, annots map[string]string) (string, string, int) { cmd := []string{ "df", } - return createSandboxContainerAndExec(t, annotations, nil, cmd) + return createSandboxContainerAndExec(t, annots, nil, cmd) } func createContainerInSandbox(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, podId, containerName, imageName string, command []string, - annotations map[string]string, mounts []*runtime.Mount, podConfig *runtime.PodSandboxConfig) string { + annots map[string]string, mounts []*runtime.Mount, podConfig *runtime.PodSandboxConfig) string { cRequest := getCreateContainerRequest(podId, containerName, imageName, command, podConfig) - cRequest.Config.Annotations = annotations + cRequest.Config.Annotations = annots cRequest.Config.Mounts = mounts containerID := createContainer(t, client, ctx, cRequest) @@ -1623,12 +1623,12 @@ func execContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx contex return output, errorMsg, exitCode } -func createSandboxContainerAndExec(t *testing.T, annotations map[string]string, mounts []*runtime.Mount, execCommand []string) (output string, errorMsg string, exitCode int) { +func createSandboxContainerAndExec(t *testing.T, annots map[string]string, mounts []*runtime.Mount, execCommand []string) (output string, errorMsg string, exitCode int) { client := newTestRuntimeClient(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annotations)) + sbRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler, WithSandboxAnnotations(annots)) podID := runPodSandbox(t, client, ctx, sbRequest) defer removePodSandbox(t, client, ctx, podID) @@ -1652,7 +1652,7 @@ func createSandboxContainerAndExec(t *testing.T, annotations map[string]string, Command: []string{ "top", }, - Annotations: annotations, + Annotations: annots, Mounts: testMounts, }, PodSandboxId: podID, @@ -1685,14 +1685,14 @@ func Test_RunPodSandbox_KernelOptions_LCOW(t *testing.T) { pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine}) - annotations := map[string]string{ - oci.AnnotationFullyPhysicallyBacked: "true", - oci.AnnotationMemorySizeInMB: "2048", - oci.AnnotationKernelBootOptions: "hugepagesz=2M hugepages=10", + annots := map[string]string{ + annotations.FullyPhysicallyBacked: "true", + annotations.MemorySizeInMB: "2048", + annotations.KernelBootOptions: "hugepagesz=2M hugepages=10", } hugePagesCmd := []string{"grep", "-i", "HugePages_Total", "/proc/meminfo"} - output, errorMsg, exitCode := createSandboxContainerAndExec(t, annotations, nil, hugePagesCmd) + output, errorMsg, exitCode := createSandboxContainerAndExec(t, annots, nil, hugePagesCmd) if exitCode != 0 { t.Fatalf("Exec into container failed with: %v and exit code: %d, %s", errorMsg, exitCode, t.Name()) diff --git a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go index 6be4ace5c5..49d1c7fab8 100644 --- a/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go +++ b/test/cri-containerd/scale_cpu_limits_to_sandbox_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/Microsoft/hcsshim/internal/oci" + "github.com/Microsoft/hcsshim/pkg/annotations" criruntime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -34,7 +34,7 @@ func Test_Scale_CPU_Limits_To_Sandbox(t *testing.T) { // set the limit to (roughly) 1 processor processorLimit := 10000 / runtime.NumCPU() contReq.Config.Annotations = map[string]string{ - oci.AnnotationContainerProcessorLimit: strconv.Itoa(processorLimit), + annotations.ContainerProcessorLimit: strconv.Itoa(processorLimit), } contID := createContainer(t, client, ctx, contReq) diff --git a/test/cri-containerd/stats_test.go b/test/cri-containerd/stats_test.go index a950fba4b0..22197b5ce0 100644 --- a/test/cri-containerd/stats_test.go +++ b/test/cri-containerd/stats_test.go @@ -4,10 +4,10 @@ package cri_containerd import ( "context" + "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "testing" - "github.com/Microsoft/hcsshim/internal/oci" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -369,9 +369,9 @@ func Test_SandboxStats_WorkingSet_PhysicallyBacked(t *testing.T) { t, test.runtimeHandler, WithSandboxAnnotations(map[string]string{ - oci.AnnotationAllowOvercommit: "false", - oci.AnnotationEnableDeferredCommit: "false", - oci.AnnotationMemorySizeInMB: sizeInMBStr, + annotations.AllowOvercommit: "false", + annotations.EnableDeferredCommit: "false", + annotations.MemorySizeInMB: sizeInMBStr, }), ) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go index cd9ca03f7e..ae1dbdeab2 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/create.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/schemaversion" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -97,13 +98,13 @@ func verifyCloneContainerSpecs(templateSpec, cloneSpec *specs.Spec) error { } // for annotations check that the values of memory & cpu annotations are same - if templateSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] != cloneSpec.Annotations[oci.AnnotationContainerMemorySizeInMB] { + if templateSpec.Annotations[annotations.ContainerMemorySizeInMB] != cloneSpec.Annotations[annotations.ContainerMemorySizeInMB] { return fmt.Errorf("memory size limit for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorCount] != cloneSpec.Annotations[oci.AnnotationContainerProcessorCount] { + if templateSpec.Annotations[annotations.ContainerProcessorCount] != cloneSpec.Annotations[annotations.ContainerProcessorCount] { return fmt.Errorf("processor count for template and clone containers can not be different") } - if templateSpec.Annotations[oci.AnnotationContainerProcessorLimit] != cloneSpec.Annotations[oci.AnnotationContainerProcessorLimit] { + if templateSpec.Annotations[annotations.ContainerProcessorLimit] != cloneSpec.Annotations[annotations.ContainerProcessorLimit] { return fmt.Errorf("processor limit for template and clone containers can not be different") } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go index b007ffdd7d..cccac4397b 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/devices.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -24,8 +25,8 @@ const deviceUtilExeName = "device-util.exe" // getSpecKernelDrivers gets any device drivers specified on the spec. // Drivers are optional, therefore do not return an error if none are on the spec. -func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { - drivers := oci.ParseAnnotationCommaSeparated(oci.AnnotationVirtualMachineKernelDrivers, annotations) +func getSpecKernelDrivers(annots map[string]string) ([]string, error) { + drivers := oci.ParseAnnotationCommaSeparated(annotations.VirtualMachineKernelDrivers, annots) for _, driver := range drivers { if _, err := os.Stat(driver); err != nil { return nil, errors.Wrapf(err, "failed to find path to drivers at %s", driver) @@ -36,8 +37,8 @@ func getSpecKernelDrivers(annotations map[string]string) ([]string, error) { // getDeviceExtensionPaths gets any device extensions paths specified on the spec. // device extensions are optional, therefore if none are on the spec, do not return an error. -func getDeviceExtensionPaths(annotations map[string]string) ([]string, error) { - extensions := oci.ParseAnnotationCommaSeparated(oci.AnnotationDeviceExtensions, annotations) +func getDeviceExtensionPaths(annots map[string]string) ([]string, error) { + extensions := oci.ParseAnnotationCommaSeparated(annotations.DeviceExtensions, annots) for _, ext := range extensions { if _, err := os.Stat(ext); err != nil { return nil, errors.Wrapf(err, "failed to find path to driver extensions at %s", ext) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go index 54c4be1eaf..8f51420540 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/hcsdoc_wcow.go @@ -20,6 +20,7 @@ import ( "github.com/Microsoft/hcsshim/internal/uvmfolder" "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -113,17 +114,17 @@ func createMountsConfig(ctx context.Context, coi *createOptionsInternal) (*mount // exclusive. func ConvertCPULimits(ctx context.Context, cid string, spec *specs.Spec, maxCPUCount int32) (int32, int32, int32, error) { cpuNumSet := 0 - cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, oci.AnnotationContainerProcessorCount, 0) + cpuCount := oci.ParseAnnotationsCPUCount(ctx, spec, annotations.ContainerProcessorCount, 0) if cpuCount > 0 { cpuNumSet++ } - cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, oci.AnnotationContainerProcessorLimit, 0) + cpuLimit := oci.ParseAnnotationsCPULimit(ctx, spec, annotations.ContainerProcessorLimit, 0) if cpuLimit > 0 { cpuNumSet++ } - cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, oci.AnnotationContainerProcessorWeight, 0) + cpuWeight := oci.ParseAnnotationsCPUWeight(ctx, spec, annotations.ContainerProcessorWeight, 0) if cpuWeight > 0 { cpuNumSet++ } @@ -244,7 +245,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Memory Resources - memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, oci.AnnotationContainerMemorySizeInMB, 0) + memoryMaxInMB := oci.ParseAnnotationsMemory(ctx, coi.Spec, annotations.ContainerMemorySizeInMB, 0) if memoryMaxInMB > 0 { v1.MemoryMaximumInMB = int64(memoryMaxInMB) v2Container.Memory = &hcsschema.Memory{ @@ -253,8 +254,8 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter } // Storage Resources - storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, oci.AnnotationContainerStorageQoSBandwidthMaximum, 0) - storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, oci.AnnotationContainerStorageQoSIopsMaximum, 0) + storageBandwidthMax := oci.ParseAnnotationsStorageBps(ctx, coi.Spec, annotations.ContainerStorageQoSBandwidthMaximum, 0) + storageIopsMax := oci.ParseAnnotationsStorageIops(ctx, coi.Spec, annotations.ContainerStorageQoSIopsMaximum, 0) if storageBandwidthMax > 0 || storageIopsMax > 0 { v1.StorageBandwidthMaximum = uint64(storageBandwidthMax) v1.StorageIOPSMaximum = uint64(storageIopsMax) @@ -392,7 +393,7 @@ func createWindowsContainerDocument(ctx context.Context, coi *createOptionsInter dumpPath = coi.HostingSystem.ProcessDumpLocation() } - if specDumpPath, ok := coi.Spec.Annotations[oci.AnnotationContainerProcessDumpLocation]; ok { + if specDumpPath, ok := coi.Spec.Annotations[annotations.ContainerProcessDumpLocation]; ok { // If a process dump path was specified at pod creation time for a hypervisor isolated pod, then // use this value. If one was specified on the container creation document then override with this // instead. Unlike Linux, Windows containers can set the dump path on a per container basis. @@ -466,8 +467,8 @@ func parseAssignedDevices(ctx context.Context, coi *createOptionsInternal, v2 *h // corresponding value the registry expects to be set. // // See DumpType at https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps for the mappings -func parseDumpType(annotations map[string]string) (int32, error) { - dmpTypeStr := annotations[oci.AnnotationWCOWProcessDumpType] +func parseDumpType(annots map[string]string) (int32, error) { + dmpTypeStr := annots[annotations.WCOWProcessDumpType] switch dmpTypeStr { case "": // If no type specified, default to full dumps. diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go index 1bdf795ae1..c4fec89f27 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/hcsoci/resources_lcow.go @@ -14,9 +14,9 @@ import ( "github.com/Microsoft/hcsshim/internal/layers" "github.com/Microsoft/hcsshim/internal/log" - "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" + "github.com/Microsoft/hcsshim/pkg/annotations" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -25,7 +25,7 @@ import ( // shim option is set. Right now we only support Nvidia gpus, so this will default to // a gpu vhd with nvidia files func getGPUVHDPath(coi *createOptionsInternal) (string, error) { - gpuVHDPath, ok := coi.Spec.Annotations[oci.AnnotationGPUVHDPath] + gpuVHDPath, ok := coi.Spec.Annotations[annotations.GPUVHDPath] if !ok || gpuVHDPath == "" { return "", fmt.Errorf("no gpu vhd specified %s", gpuVHDPath) } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go deleted file mode 100644 index bed5230470..0000000000 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/annotations.go +++ /dev/null @@ -1,237 +0,0 @@ -package oci - -const ( - // AnnotationContainerMemorySizeInMB overrides the container memory size set - // via the OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Memory.Limit`. - AnnotationContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" - - // AnnotationContainerProcessorCount overrides the container processor count - // set via the OCI spec. - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. - AnnotationContainerProcessorCount = "io.microsoft.container.processor.count" - - // AnnotationContainerProcessorLimit overrides the container processor limit - // set via the OCI spec. - // - // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.CPU.Maximum`. - AnnotationContainerProcessorLimit = "io.microsoft.container.processor.limit" - - // AnnotationContainerProcessorWeight overrides the container processor - // weight set via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default) - // - // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually - // exclusive and the caller MUST only set one of the values. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. - AnnotationContainerProcessorWeight = "io.microsoft.container.processor.weight" - - // AnnotationContainerStorageQoSBandwidthMaximum overrides the container - // storage bandwidth per second set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Bps`. - AnnotationContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" - - // AnnotationContainerStorageQoSIopsMaximum overrides the container storage - // maximum iops set via the OCI spec. - // - // Note: This is only present because CRI does not (currently) have a - // `WindowsPodSandboxConfig` for setting this correctly. It should not be - // used via OCI runtimes and rather use - // `spec.Windows.Resources.Storage.Iops`. - AnnotationContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" - - // AnnotationGPUVHDPath overrides the default path to search for the gpu vhd - AnnotationGPUVHDPath = "io.microsoft.lcow.gpuvhdpath" - - // AnnotationVirtualMachineKernelDrivers indicates what drivers to install in the pod. - // This value should contain a list of comma separated directories containing all - // files and information needed to install given driver(s). For windows, this may - // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. - // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. - AnnotationVirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" - - // AnnotationDeviceExtensions contains a comma separated list of full paths to device extension files. - // The content of these are added to a container's hcs create document. - AnnotationDeviceExtensions = "io.microsoft.container.wcow.deviceextensions" - - // AnnotationHostProcessInheritUser indicates whether to ignore the username passed in to run a host process - // container as and instead inherit the user token from the executable that is launching the container process. - AnnotationHostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" - - // AnnotationHostProcessContainer indicates to launch a host process container (job container in this repository). - AnnotationHostProcessContainer = "microsoft.com/hostprocess-container" - - // AnnotationAllowOvercommit indicates if we should allow over commit memory for UVM. - // Defaults to true. For physical backed memory, set to false. - AnnotationAllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" - - // AnnotationEnableDeferredCommit indicates if we should allow deferred memory commit for UVM. - // Defaults to false. For virtual memory with deferred commit, set to true. - AnnotationEnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" - - // AnnotationEnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM - // to trim non-zeroed pages from the working set (if supported by the guest operating system). - AnnotationEnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" - - // AnnotationMemorySizeInMB overrides the container memory size set via the - // OCI spec. - // - // Note: This annotation is in MB. OCI is in Bytes. When using this override - // the caller MUST use MB or sizing will be wrong. - AnnotationMemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" - - // AnnotationMemoryLowMMIOGapInMB indicates the low MMIO gap in MB - AnnotationMemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO base in MB - AnnotationMemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" - - // AnnotationMemoryHighMMIOBaseInMB indicates the high MMIO gap in MB - AnnotationMemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" - - // annotationProcessorCount overrides the hypervisor isolated vCPU count set - // via the OCI spec. - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" - - // annotationProcessorLimit overrides the hypervisor isolated vCPU limit set - // via the OCI spec. - // - // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the - // default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" - - // AnnotationProcessorWeight overrides the hypervisor isolated vCPU weight set - // via the OCI spec. - // - // Weight allows values 0 - 10,000. (100 is the default if omitted) - // - // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on - // the UVM are not mutually exclusive and can be set together. - AnnotationProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" - - // AnnotationVPMemCount indicates the max number of vpmem devices that can be used on the UVM - AnnotationVPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" - - // AnnotationVPMemSize indicates the size of the VPMem devices. - AnnotationVPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" - - // AnnotationPreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. - // valid values are "initrd" or "vhd" - AnnotationPreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" - - // AnnotationBootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM - AnnotationBootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" - - // AnnotationKernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` - AnnotationKernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" - - // AnnotationVPCIEnabled indicates that pci support should be enabled for the LCOW UVM - AnnotationVPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" - - // AnnotationVPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping - AnnotationVPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" - - // AnnotationKernelBootOptions is used to specify kernel options used while booting a linux kernel - AnnotationKernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" - - // AnnotationStorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` - // will default to the platform default. - AnnotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" - - // AnnotationStorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will - // default to the platform default. - AnnotationStorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" - - // AnnotationFullyPhysicallyBacked indicates that the UVM should use physically backed memory only, - // including for additional devices added later. - AnnotationFullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" - - // AnnotationDisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM - // for WCOW. - AnnotationDisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" - - // AnnotationVSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM - AnnotationVSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" - - // AnnotationCPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any - AnnotationCPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" - - // AnnotationSaveAsTemplate annotation must be used with a pod & container creation request. - // If this annotation is present in the request then it will save the UVM (pod) - // and the container(s) inside it as a template. However, this also means that this - // pod and the containers inside this pod will permananetly stay in the - // paused/templated state and can not be resumed again. - AnnotationSaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" - - // AnnotationTemplateID should be used when creating a pod or a container from a template. - // When creating a pod from a template use the ID of the templated pod as the - // TemplateID and when creating a container use the ID of the templated container as - // the TemplateID. It is the client's responsibility to make sure that the sandbox - // within which a cloned container needs to be created must also be created from the - // same template. - AnnotationTemplateID = "io.microsoft.virtualmachine.templateid" - - // AnnotationNetworkConfigProxy holds the address of the network config proxy service. - // If set, network setup will be attempted via ncproxy. - AnnotationNetworkConfigProxy = "io.microsoft.network.ncproxy" - - // AnnotationNcproxyContainerID indicates whether or not to use the hcsshim container ID - // when setting up ncproxy and computeagent - AnnotationNcproxyContainerID = "io.microsoft.network.ncproxy.containerid" - - // AnnotationEncryptedScratchDisk indicates whether or not the container scratch disks - // should be encrypted or not - AnnotationEncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" - - // AnnotationSecurityPolicy is used to specify a security policy for opengcs to enforce - AnnotationSecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" - // AnnotationContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As - // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of - // some kind (vhd, bind mount, fileshare mount etc.) - AnnotationContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" - - // AnnotationWCOWProcessDumpType specifies the type of dump to create when generating a local user mode - // process dump for Windows containers. The supported options are "mini", and "full". - // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps - AnnotationWCOWProcessDumpType = "io.microsoft.wcow.processdumptype" - - // AnnotationRLimitCore specifies the core rlimit value for a container. This will need to be set - // in order to have core dumps generated for a given container. - AnnotationRLimitCore = "io.microsoft.lcow.rlimitcore" -) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go index 3c3f89280d..569b035654 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/sandbox.go @@ -2,28 +2,22 @@ package oci import ( "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" ) -// KubernetesContainerTypeAnnotation is the annotation used by CRI to define the `ContainerType`. -const KubernetesContainerTypeAnnotation = "io.kubernetes.cri.container-type" - -// KubernetesSandboxIDAnnotation is the annotation used by CRI to define the -// KubernetesContainerTypeAnnotation == "sandbox"` ID. -const KubernetesSandboxIDAnnotation = "io.kubernetes.cri.sandbox-id" - // KubernetesContainerType defines the valid types of the -// `KubernetesContainerTypeAnnotation` annotation. +// `annotations.KubernetesContainerType` annotation. type KubernetesContainerType string const ( // KubernetesContainerTypeNone is only valid when - // `KubernetesContainerTypeAnnotation` is not set. + // `annotations.KubernetesContainerType` is not set. KubernetesContainerTypeNone KubernetesContainerType = "" // KubernetesContainerTypeContainer is valid when - // `KubernetesContainerTypeAnnotation == "container"`. + // `annotations.KubernetesContainerType == "container"`. KubernetesContainerTypeContainer KubernetesContainerType = "container" // KubernetesContainerTypeSandbox is valid when - // `KubernetesContainerTypeAnnotation == "sandbox"`. + // `annotations.KubernetesContainerType == "sandbox"`. KubernetesContainerTypeSandbox KubernetesContainerType = "sandbox" ) @@ -32,27 +26,27 @@ const ( // annotations and if found validates the set before returning. func GetSandboxTypeAndID(specAnnotations map[string]string) (KubernetesContainerType, string, error) { var ct KubernetesContainerType - if t, ok := specAnnotations[KubernetesContainerTypeAnnotation]; ok { + if t, ok := specAnnotations[annotations.KubernetesContainerType]; ok { switch t { case string(KubernetesContainerTypeContainer): ct = KubernetesContainerTypeContainer case string(KubernetesContainerTypeSandbox): ct = KubernetesContainerTypeSandbox default: - return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", KubernetesContainerTypeAnnotation, t) + return KubernetesContainerTypeNone, "", fmt.Errorf("invalid '%s': '%s'", annotations.KubernetesContainerType, t) } } - id := specAnnotations[KubernetesSandboxIDAnnotation] + id := specAnnotations[annotations.KubernetesSandboxID] switch ct { case KubernetesContainerTypeContainer, KubernetesContainerTypeSandbox: if id == "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesContainerTypeAnnotation, KubernetesSandboxIDAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesContainerType, annotations.KubernetesSandboxID) } default: if id != "" { - return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", KubernetesSandboxIDAnnotation, KubernetesContainerTypeAnnotation) + return KubernetesContainerTypeNone, "", fmt.Errorf("cannot specify '%s' without '%s'", annotations.KubernetesSandboxID, annotations.KubernetesContainerType) } } return ct, id, nil diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go index 85f52016ff..6452241eca 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/util.go @@ -1,6 +1,9 @@ package oci -import "github.com/opencontainers/runtime-spec/specs-go" +import ( + "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/opencontainers/runtime-spec/specs-go" +) // IsLCOW checks if `s` is a LCOW config. func IsLCOW(s *specs.Spec) bool { @@ -19,5 +22,5 @@ func IsIsolated(s *specs.Spec) bool { // IsJobContainer checks if `s` is asking for a Windows job container. func IsJobContainer(s *specs.Spec) bool { - return s.Annotations[AnnotationHostProcessContainer] == "true" + return s.Annotations[annotations.HostProcessContainer] == "true" } diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go index c77a90731a..fcf41fba08 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/oci/uvm.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/Microsoft/hcsshim/pkg/annotations" "strconv" "strings" @@ -219,13 +220,13 @@ func parseAnnotationsString(a map[string]string, key string, def string) string // if this create request should be considered as a template creation request. If value // is found the returns the actual value, returns false otherwise. func ParseAnnotationsSaveAsTemplate(ctx context.Context, s *specs.Spec) bool { - return parseAnnotationsBool(ctx, s.Annotations, AnnotationSaveAsTemplate, false) + return parseAnnotationsBool(ctx, s.Annotations, annotations.SaveAsTemplate, false) } // ParseAnnotationsTemplateID searches for the templateID in the create request. If the // value is found then returns the value otherwise returns the empty string. func ParseAnnotationsTemplateID(ctx context.Context, s *specs.Spec) string { - return parseAnnotationsString(s.Annotations, AnnotationTemplateID, "") + return parseAnnotationsString(s.Annotations, annotations.TemplateID, "") } func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, templateID string, err error) { @@ -244,7 +245,7 @@ func ParseCloneAnnotations(ctx context.Context, s *specs.Spec) (isTemplate bool, // handleAnnotationKernelDirectBoot handles parsing annotationKernelDirectBoot and setting // implied annotations from the result. func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.KernelDirect = parseAnnotationsBool(ctx, a, AnnotationKernelDirectBoot, lopts.KernelDirect) + lopts.KernelDirect = parseAnnotationsBool(ctx, a, annotations.KernelDirectBoot, lopts.KernelDirect) if !lopts.KernelDirect { lopts.KernelFile = uvm.KernelFile } @@ -253,7 +254,7 @@ func handleAnnotationKernelDirectBoot(ctx context.Context, a map[string]string, // handleAnnotationPreferredRootFSType handles parsing annotationPreferredRootFSType and setting // implied annotations from the result func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]string, lopts *uvm.OptionsLCOW) { - lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, AnnotationPreferredRootFSType, lopts.PreferredRootFSType) + lopts.PreferredRootFSType = parseAnnotationsPreferredRootFSType(ctx, a, annotations.PreferredRootFSType, lopts.PreferredRootFSType) switch lopts.PreferredRootFSType { case uvm.PreferredRootFSTypeInitRd: lopts.RootFSFile = uvm.InitrdFile @@ -267,7 +268,7 @@ func handleAnnotationPreferredRootFSType(ctx context.Context, a map[string]strin func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]string, opts interface{}) { switch options := opts.(type) { case *uvm.OptionsLCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false options.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd @@ -275,7 +276,7 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str options.VPMemDeviceCount = 0 } case *uvm.OptionsWCOW: - options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, AnnotationFullyPhysicallyBacked, options.FullyPhysicallyBacked) + options.FullyPhysicallyBacked = parseAnnotationsBool(ctx, a, annotations.FullyPhysicallyBacked, options.FullyPhysicallyBacked) if options.FullyPhysicallyBacked { options.AllowOvercommit = false } @@ -285,8 +286,8 @@ func handleAnnotationFullyPhysicallyBacked(ctx context.Context, a map[string]str // handleCloneAnnotations handles parsing annotations related to template creation and cloning // Since late cloning is only supported for WCOW this function only deals with WCOW options. func handleCloneAnnotations(ctx context.Context, a map[string]string, wopts *uvm.OptionsWCOW) (err error) { - wopts.IsTemplate = parseAnnotationsBool(ctx, a, AnnotationSaveAsTemplate, false) - templateID := parseAnnotationsString(a, AnnotationTemplateID, "") + wopts.IsTemplate = parseAnnotationsBool(ctx, a, annotations.SaveAsTemplate, false) + templateID := parseAnnotationsString(a, annotations.TemplateID, "") if templateID != "" { tc, err := clone.FetchTemplateConfig(ctx, templateID) if err != nil { @@ -310,29 +311,29 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( } if IsLCOW(s) { lopts := uvm.NewDefaultOptionsLCOW(id, owner) - lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, lopts.MemorySizeInMB) - lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) - lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) - lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) - lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, lopts.AllowOvercommit) - lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, lopts.EnableDeferredCommit) - lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableColdDiscardHint, lopts.EnableColdDiscardHint) - lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, lopts.ProcessorCount) - lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, lopts.ProcessorLimit) - lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, lopts.ProcessorWeight) - lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, AnnotationVPMemCount, lopts.VPMemDeviceCount) - lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, AnnotationVPMemSize, lopts.VPMemSizeBytes) - lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPMemNoMultiMapping, lopts.VPMemNoMultiMapping) - lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) - lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) - lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, AnnotationVPCIEnabled, lopts.VPCIEnabled) - lopts.BootFilesPath = parseAnnotationsString(s.Annotations, AnnotationBootFilesRootPath, lopts.BootFilesPath) - lopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, lopts.CPUGroupID) - lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, lopts.NetworkConfigProxy) - lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, AnnotationEncryptedScratchDisk, lopts.EnableScratchEncryption) - lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, AnnotationSecurityPolicy, lopts.SecurityPolicy) - lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, AnnotationKernelBootOptions, lopts.KernelBootOptions) - lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, lopts.ProcessDumpLocation) + lopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, lopts.MemorySizeInMB) + lopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, lopts.LowMMIOGapInMB) + lopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, lopts.HighMMIOBaseInMB) + lopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, lopts.HighMMIOGapInMB) + lopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, lopts.AllowOvercommit) + lopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, lopts.EnableDeferredCommit) + lopts.EnableColdDiscardHint = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableColdDiscardHint, lopts.EnableColdDiscardHint) + lopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, lopts.ProcessorCount) + lopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, lopts.ProcessorLimit) + lopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, lopts.ProcessorWeight) + lopts.VPMemDeviceCount = parseAnnotationsUint32(ctx, s.Annotations, annotations.VPMemCount, lopts.VPMemDeviceCount) + lopts.VPMemSizeBytes = parseAnnotationsUint64(ctx, s.Annotations, annotations.VPMemSize, lopts.VPMemSizeBytes) + lopts.VPMemNoMultiMapping = parseAnnotationsBool(ctx, s.Annotations, annotations.VPMemNoMultiMapping, lopts.VPMemNoMultiMapping) + lopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, lopts.StorageQoSBandwidthMaximum) + lopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, lopts.StorageQoSIopsMaximum) + lopts.VPCIEnabled = parseAnnotationsBool(ctx, s.Annotations, annotations.VPCIEnabled, lopts.VPCIEnabled) + lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotations.BootFilesRootPath, lopts.BootFilesPath) + lopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, lopts.CPUGroupID) + lopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, lopts.NetworkConfigProxy) + lopts.EnableScratchEncryption = parseAnnotationsBool(ctx, s.Annotations, annotations.EncryptedScratchDisk, lopts.EnableScratchEncryption) + lopts.SecurityPolicy = parseAnnotationsString(s.Annotations, annotations.SecurityPolicy, lopts.SecurityPolicy) + lopts.KernelBootOptions = parseAnnotationsString(s.Annotations, annotations.KernelBootOptions, lopts.KernelBootOptions) + lopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, lopts.ProcessDumpLocation) handleAnnotationPreferredRootFSType(ctx, s.Annotations, lopts) handleAnnotationKernelDirectBoot(ctx, s.Annotations, lopts) @@ -342,22 +343,22 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( return lopts, nil } else if IsWCOW(s) { wopts := uvm.NewDefaultOptionsWCOW(id, owner) - wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, AnnotationMemorySizeInMB, wopts.MemorySizeInMB) - wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) - wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) - wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, AnnotationMemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) - wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationAllowOvercommit, wopts.AllowOvercommit) - wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, AnnotationEnableDeferredCommit, wopts.EnableDeferredCommit) - wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, AnnotationProcessorCount, wopts.ProcessorCount) - wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, AnnotationProcessorLimit, wopts.ProcessorLimit) - wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, AnnotationProcessorWeight, wopts.ProcessorWeight) - wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, AnnotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) - wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, AnnotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) - wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, AnnotationDisableCompartmentNamespace, wopts.DisableCompartmentNamespace) - wopts.CPUGroupID = parseAnnotationsString(s.Annotations, AnnotationCPUGroupID, wopts.CPUGroupID) - wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, AnnotationNetworkConfigProxy, wopts.NetworkConfigProxy) - wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, AnnotationVSMBNoDirectMap, wopts.NoDirectMap) - wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, AnnotationContainerProcessDumpLocation, wopts.ProcessDumpLocation) + wopts.MemorySizeInMB = ParseAnnotationsMemory(ctx, s, annotations.MemorySizeInMB, wopts.MemorySizeInMB) + wopts.LowMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryLowMMIOGapInMB, wopts.LowMMIOGapInMB) + wopts.HighMMIOBaseInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOBaseInMB, wopts.HighMMIOBaseInMB) + wopts.HighMMIOGapInMB = parseAnnotationsUint64(ctx, s.Annotations, annotations.MemoryHighMMIOGapInMB, wopts.HighMMIOGapInMB) + wopts.AllowOvercommit = parseAnnotationsBool(ctx, s.Annotations, annotations.AllowOvercommit, wopts.AllowOvercommit) + wopts.EnableDeferredCommit = parseAnnotationsBool(ctx, s.Annotations, annotations.EnableDeferredCommit, wopts.EnableDeferredCommit) + wopts.ProcessorCount = ParseAnnotationsCPUCount(ctx, s, annotations.ProcessorCount, wopts.ProcessorCount) + wopts.ProcessorLimit = ParseAnnotationsCPULimit(ctx, s, annotations.ProcessorLimit, wopts.ProcessorLimit) + wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotations.ProcessorWeight, wopts.ProcessorWeight) + wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotations.StorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum) + wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotations.StorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum) + wopts.DisableCompartmentNamespace = parseAnnotationsBool(ctx, s.Annotations, annotations.DisableCompartmentNamespace, wopts.DisableCompartmentNamespace) + wopts.CPUGroupID = parseAnnotationsString(s.Annotations, annotations.CPUGroupID, wopts.CPUGroupID) + wopts.NetworkConfigProxy = parseAnnotationsString(s.Annotations, annotations.NetworkConfigProxy, wopts.NetworkConfigProxy) + wopts.NoDirectMap = parseAnnotationsBool(ctx, s.Annotations, annotations.VSMBNoDirectMap, wopts.NoDirectMap) + wopts.ProcessDumpLocation = parseAnnotationsString(s.Annotations, annotations.ContainerProcessDumpLocation, wopts.ProcessDumpLocation) handleAnnotationFullyPhysicallyBacked(ctx, s.Annotations, wopts) if err := handleCloneAnnotations(ctx, s.Annotations, wopts); err != nil { return nil, err @@ -374,24 +375,24 @@ func UpdateSpecFromOptions(s specs.Spec, opts *runhcsopts.Options) specs.Spec { return s } - if _, ok := s.Annotations[AnnotationBootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { - s.Annotations[AnnotationBootFilesRootPath] = opts.BootFilesRootPath + if _, ok := s.Annotations[annotations.BootFilesRootPath]; !ok && opts.BootFilesRootPath != "" { + s.Annotations[annotations.BootFilesRootPath] = opts.BootFilesRootPath } - if _, ok := s.Annotations[AnnotationProcessorCount]; !ok && opts.VmProcessorCount != 0 { - s.Annotations[AnnotationProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) + if _, ok := s.Annotations[annotations.ProcessorCount]; !ok && opts.VmProcessorCount != 0 { + s.Annotations[annotations.ProcessorCount] = strconv.FormatInt(int64(opts.VmProcessorCount), 10) } - if _, ok := s.Annotations[AnnotationMemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { - s.Annotations[AnnotationMemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) + if _, ok := s.Annotations[annotations.MemorySizeInMB]; !ok && opts.VmMemorySizeInMb != 0 { + s.Annotations[annotations.MemorySizeInMB] = strconv.FormatInt(int64(opts.VmMemorySizeInMb), 10) } - if _, ok := s.Annotations[AnnotationGPUVHDPath]; !ok && opts.GPUVHDPath != "" { - s.Annotations[AnnotationGPUVHDPath] = opts.GPUVHDPath + if _, ok := s.Annotations[annotations.GPUVHDPath]; !ok && opts.GPUVHDPath != "" { + s.Annotations[annotations.GPUVHDPath] = opts.GPUVHDPath } - if _, ok := s.Annotations[AnnotationNetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { - s.Annotations[AnnotationNetworkConfigProxy] = opts.NCProxyAddr + if _, ok := s.Annotations[annotations.NetworkConfigProxy]; !ok && opts.NCProxyAddr != "" { + s.Annotations[annotations.NetworkConfigProxy] = opts.NCProxyAddr } return s diff --git a/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go new file mode 100644 index 0000000000..0042b3d24c --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/pkg/annotations/annotations.go @@ -0,0 +1,253 @@ +package annotations + +const ( + // ContainerMemorySizeInMB overrides the container memory size set + // via the OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Memory.Limit`. + ContainerMemorySizeInMB = "io.microsoft.container.memory.sizeinmb" + + // ContainerProcessorCount overrides the container processor count + // set via the OCI spec. + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Count`. + ContainerProcessorCount = "io.microsoft.container.processor.count" + + // ContainerProcessorLimit overrides the container processor limit + // set via the OCI spec. + // + // Limit allows values 1 - 10,000 where 10,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.CPU.Maximum`. + ContainerProcessorLimit = "io.microsoft.container.processor.limit" + + // ContainerProcessorWeight overrides the container processor + // weight set via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default) + // + // Note: For Windows Process Containers CPU Count/Limit/Weight are mutually + // exclusive and the caller MUST only set one of the values. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use `spec.Windows.Resources.CPU.Shares`. + ContainerProcessorWeight = "io.microsoft.container.processor.weight" + + // ContainerStorageQoSBandwidthMaximum overrides the container + // storage bandwidth per second set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Bps`. + ContainerStorageQoSBandwidthMaximum = "io.microsoft.container.storage.qos.bandwidthmaximum" + + // ContainerStorageQoSIopsMaximum overrides the container storage + // maximum iops set via the OCI spec. + // + // Note: This is only present because CRI does not (currently) have a + // `WindowsPodSandboxConfig` for setting this correctly. It should not be + // used via OCI runtimes and rather use + // `spec.Windows.Resources.Storage.Iops`. + ContainerStorageQoSIopsMaximum = "io.microsoft.container.storage.qos.iopsmaximum" + + // GPUVHDPath overrides the default path to search for the gpu vhd + GPUVHDPath = "io.microsoft.lcow.gpuvhdpath" + + // ContainerGPUCapabilities is used to find the gpu capabilities on the container spec + ContainerGPUCapabilities = "io.microsoft.container.gpu.capabilities" + + // VirtualMachineKernelDrivers indicates what drivers to install in the pod. + // This value should contain a list of comma separated directories containing all + // files and information needed to install given driver(s). For windows, this may + // include .sys, .inf, .cer, and/or other files used during standard installation with pnputil. + // For LCOW, this may include a vhd file that contains kernel modules as *.ko files. + VirtualMachineKernelDrivers = "io.microsoft.virtualmachine.kerneldrivers" + + // DeviceExtensions contains a comma separated list of full paths to device extension files. + // The content of these are added to a container's hcs create document. + DeviceExtensions = "io.microsoft.container.wcow.deviceextensions" + + // HostProcessInheritUser indicates whether to ignore the username passed in to run a host process + // container as and instead inherit the user token from the executable that is launching the container process. + HostProcessInheritUser = "microsoft.com/hostprocess-inherit-user" + + // HostProcessContainer indicates to launch a host process container (job container in this repository). + HostProcessContainer = "microsoft.com/hostprocess-container" + + // AllowOvercommit indicates if we should allow over commit memory for UVM. + // Defaults to true. For physical backed memory, set to false. + AllowOvercommit = "io.microsoft.virtualmachine.computetopology.memory.allowovercommit" + + // EnableDeferredCommit indicates if we should allow deferred memory commit for UVM. + // Defaults to false. For virtual memory with deferred commit, set to true. + EnableDeferredCommit = "io.microsoft.virtualmachine.computetopology.memory.enabledeferredcommit" + + // EnableColdDiscardHint indicates whether to enable cold discard hint, which allows the UVM + // to trim non-zeroed pages from the working set (if supported by the guest operating system). + EnableColdDiscardHint = "io.microsoft.virtualmachine.computetopology.memory.enablecolddiscardhint" + + // MemorySizeInMB overrides the container memory size set via the + // OCI spec. + // + // Note: This annotation is in MB. OCI is in Bytes. When using this override + // the caller MUST use MB or sizing will be wrong. + MemorySizeInMB = "io.microsoft.virtualmachine.computetopology.memory.sizeinmb" + + // MemoryLowMMIOGapInMB indicates the low MMIO gap in MB + MemoryLowMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.lowmmiogapinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO base in MB + MemoryHighMMIOBaseInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiobaseinmb" + + // MemoryHighMMIOBaseInMB indicates the high MMIO gap in MB + MemoryHighMMIOGapInMB = "io.microsoft.virtualmachine.computetopology.memory.highmmiogapinmb" + + // ProcessorCount overrides the hypervisor isolated vCPU count set + // via the OCI spec. + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorCount = "io.microsoft.virtualmachine.computetopology.processor.count" + + // ProcessorLimit overrides the hypervisor isolated vCPU limit set + // via the OCI spec. + // + // Limit allows values 1 - 100,000 where 100,000 means 100% CPU. (And is the + // default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorLimit = "io.microsoft.virtualmachine.computetopology.processor.limit" + + // ProcessorWeight overrides the hypervisor isolated vCPU weight set + // via the OCI spec. + // + // Weight allows values 0 - 10,000. (100 is the default if omitted) + // + // Note: Unlike Windows process isolated container QoS Count/Limt/Weight on + // the UVM are not mutually exclusive and can be set together. + ProcessorWeight = "io.microsoft.virtualmachine.computetopology.processor.weight" + + // VPMemCount indicates the max number of vpmem devices that can be used on the UVM + VPMemCount = "io.microsoft.virtualmachine.devices.virtualpmem.maximumcount" + + // VPMemSize indicates the size of the VPMem devices. + VPMemSize = "io.microsoft.virtualmachine.devices.virtualpmem.maximumsizebytes" + + // PreferredRootFSType indicates what the preferred rootfs type should be for an LCOW UVM. + // valid values are "initrd" or "vhd" + PreferredRootFSType = "io.microsoft.virtualmachine.lcow.preferredrootfstype" + + // BootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM + BootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" + + // KernelDirectBoot indicates that we should skip UEFI and boot directly to `kernel` + KernelDirectBoot = "io.microsoft.virtualmachine.lcow.kerneldirectboot" + + // VPCIEnabled indicates that pci support should be enabled for the LCOW UVM + VPCIEnabled = "io.microsoft.virtualmachine.lcow.vpcienabled" + + // VPMemNoMultiMapping indicates that we should disable LCOW vpmem layer multi mapping + VPMemNoMultiMapping = "io.microsoft.virtualmachine.lcow.vpmem.nomultimapping" + + // KernelBootOptions is used to specify kernel options used while booting a linux kernel + KernelBootOptions = "io.microsoft.virtualmachine.lcow.kernelbootoptions" + + // StorageQoSBandwidthMaximum indicates the maximum number of bytes per second. If `0` + // will default to the platform default. + StorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum" + + // StorageQoSIopsMaximum indicates the maximum number of Iops. If `0` will + // default to the platform default. + StorageQoSIopsMaximum = "io.microsoft.virtualmachine.storageqos.iopsmaximum" + + // FullyPhysicallyBacked indicates that the UVM should use physically backed memory only, + // including for additional devices added later. + FullyPhysicallyBacked = "io.microsoft.virtualmachine.fullyphysicallybacked" + + // DisableCompartmentNamespace sets whether to disable namespacing the network compartment in the UVM + // for WCOW. + DisableCompartmentNamespace = "io.microsoft.virtualmachine.disablecompartmentnamespace" + + // VSMBNoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM + VSMBNoDirectMap = "io.microsoft.virtualmachine.wcow.virtualSMB.nodirectmap" + + // CPUGroupID specifies the cpugroup ID that a UVM should be assigned to if any + CPUGroupID = "io.microsoft.virtualmachine.cpugroup.id" + + // SaveAsTemplate annotation must be used with a pod & container creation request. + // If this annotation is present in the request then it will save the UVM (pod) + // and the container(s) inside it as a template. However, this also means that this + // pod and the containers inside this pod will permananetly stay in the + // paused/templated state and can not be resumed again. + SaveAsTemplate = "io.microsoft.virtualmachine.saveastemplate" + + // TemplateID should be used when creating a pod or a container from a template. + // When creating a pod from a template use the ID of the templated pod as the + // TemplateID and when creating a container use the ID of the templated container as + // the TemplateID. It is the client's responsibility to make sure that the sandbox + // within which a cloned container needs to be created must also be created from the + // same template. + TemplateID = "io.microsoft.virtualmachine.templateid" + + // NetworkConfigProxy holds the address of the network config proxy service. + // If set, network setup will be attempted via ncproxy. + NetworkConfigProxy = "io.microsoft.network.ncproxy" + + // NcproxyContainerID indicates whether or not to use the hcsshim container ID + // when setting up ncproxy and computeagent + NcproxyContainerID = "io.microsoft.network.ncproxy.containerid" + + // EncryptedScratchDisk indicates whether or not the container scratch disks + // should be encrypted or not + EncryptedScratchDisk = "io.microsoft.virtualmachine.storage.scratch.encrypted" + + // SecurityPolicy is used to specify a security policy for opengcs to enforce + SecurityPolicy = "io.microsoft.virtualmachine.lcow.securitypolicy" + // ContainerProcessDumpLocation specifies a path inside of containers to save process dumps to. As + // the scratch space for a container is generally cleaned up after exit, this is best set to a volume mount of + // some kind (vhd, bind mount, fileshare mount etc.) + ContainerProcessDumpLocation = "io.microsoft.container.processdumplocation" + + // WCOWProcessDumpType specifies the type of dump to create when generating a local user mode + // process dump for Windows containers. The supported options are "mini", and "full". + // See DumpType: https://docs.microsoft.com/en-us/windows/win32/wer/collecting-user-mode-dumps + WCOWProcessDumpType = "io.microsoft.wcow.processdumptype" + + // RLimitCore specifies the core rlimit value for a container. This will need to be set + // in order to have core dumps generated for a given container. + RLimitCore = "io.microsoft.lcow.rlimitcore" + + // LCOWDevShmSizeInKb specifies the size of LCOW /dev/shm. + LCOWDevShmSizeInKb = "io.microsoft.lcow.shm.size-kb" + + // LCOWPrivileged is used to specify that the container should be run in privileged mode + LCOWPrivileged = "io.microsoft.virtualmachine.lcow.privileged" + + // KubernetesContainerType is the annotation used by CRI to define the `ContainerType`. + KubernetesContainerType = "io.kubernetes.cri.container-type" + + // KubernetesSandboxID is the annotation used by CRI to define the + // KubernetesContainerType == "sandbox"` ID. + KubernetesSandboxID = "io.kubernetes.cri.sandbox-id" +) diff --git a/test/vendor/modules.txt b/test/vendor/modules.txt index 40a7a888f4..27c7647555 100644 --- a/test/vendor/modules.txt +++ b/test/vendor/modules.txt @@ -63,6 +63,7 @@ github.com/Microsoft/hcsshim/internal/wclayer github.com/Microsoft/hcsshim/internal/wcow github.com/Microsoft/hcsshim/internal/winapi github.com/Microsoft/hcsshim/osversion +github.com/Microsoft/hcsshim/pkg/annotations github.com/Microsoft/hcsshim/pkg/go-runhcs github.com/Microsoft/hcsshim/pkg/ociwclayer github.com/Microsoft/hcsshim/pkg/octtrpc