diff --git a/config-linux.md b/config-linux.md index f73d893c9..f1685f9ad 100644 --- a/config-linux.md +++ b/config-linux.md @@ -169,7 +169,7 @@ In addition to any devices configured with this setting, the runtime MUST also s ## Control groups Also known as cgroups, they are used to restrict resource usage for a container and handle device access. -cgroups provide controls (through controllers) to restrict cpu, memory, IO, pids and network for the container. +cgroups provide controls (through controllers) to restrict cpu, memory, IO, pids, network and RDMA resources for the container. For more information, see the [kernel cgroups documentation][cgroup-v1]. ### Cgroups Path @@ -455,6 +455,36 @@ The following parameters can be specified to set up the controller: } ``` +### RDMA + +**`rdmaLimits`** (object, OPTIONAL) represents the cgroup subsystem `rdma`. +For more information, see the kernel cgroups documentation about [rdma][cgroup-v1-rdma]. + +The name of the device to limit is the entry key. +Entry values are objects with the following properties: + +* **`hca_handles`** *(uint32, OPTIONAL)* - specifies the maximum number of hca_objects in the cgroup +* **`hca_objects`** *(uint32, OPTIONAL)* - specifies the maximum number of hca_handles in the cgroup + +You MUST specify at least one of the `hca_handles` or `hca_objects` in a given entry, and MAY specify both. + +#### Example + +```json +"rdmaLimits": { + "mlx5_1": { + "hca_handles": 3, + "hca_objects": 10000 + }, + "mlx4_0": { + "hca_objects": 1000 + }, + "rxe3": { + "hca_objects": 10000 + } +} +``` + ## IntelRdt **`intelRdt`** (object, OPTIONAL) represents the [Intel Resource Director Technology][intel-rdt-cat-kernel-interface]. @@ -647,6 +677,7 @@ The following parameters can be specified to set up seccomp: [cgroup-v1-net-cls]: https://www.kernel.org/doc/Documentation/cgroup-v1/net_cls.txt [cgroup-v1-net-prio]: https://www.kernel.org/doc/Documentation/cgroup-v1/net_prio.txt [cgroup-v1-pids]: https://www.kernel.org/doc/Documentation/cgroup-v1/pids.txt +[cgroup-v1-rdma]: https://www.kernel.org/doc/Documentation/cgroup-v1/rdma.txt [cgroup-v2]: https://www.kernel.org/doc/Documentation/cgroup-v2.txt [devices]: https://www.kernel.org/doc/Documentation/admin-guide/devices.txt [devpts]: https://www.kernel.org/doc/Documentation/filesystems/devpts.txt diff --git a/schema/config-linux.json b/schema/config-linux.json index 83a562677..619884ab3 100644 --- a/schema/config-linux.json +++ b/schema/config-linux.json @@ -212,6 +212,12 @@ } } } + }, + "rdmaLimits": { + "type": "object", + "additionalProperties": { + "$ref": "defs-linux.json#/definitions/RdmaLimit" + } } } }, diff --git a/schema/defs-linux.json b/schema/defs-linux.json index 4d9620a4a..ead4c1dc2 100644 --- a/schema/defs-linux.json +++ b/schema/defs-linux.json @@ -240,6 +240,17 @@ "priority" ] }, + "RdmaLimit": { + "type": "object", + "properties": { + "hca_handles": { + "$ref": "defs.json#/definitions/uint32" + }, + "hca_objects": { + "$ref": "defs.json#/definitions/uint32" + } + } + }, "NamespaceType": { "type": "string", "enum": [ diff --git a/schema/test/config/bad/linux-rdma.json b/schema/test/config/bad/linux-rdma.json new file mode 100644 index 000000000..c041f460a --- /dev/null +++ b/schema/test/config/bad/linux-rdma.json @@ -0,0 +1,15 @@ +{ + "ociVersion": "1.0.0", + "root": { + "path": "rootfs" + }, + "linux": { + "resources": { + "rdmaLimits": { + "mlx5_1": { + "hca_handles": "not a uint32" + } + } + } + } +} diff --git a/schema/test/config/good/linux-rdma.json b/schema/test/config/good/linux-rdma.json new file mode 100644 index 000000000..1a91577b0 --- /dev/null +++ b/schema/test/config/good/linux-rdma.json @@ -0,0 +1,22 @@ +{ + "ociVersion": "1.0.0", + "root": { + "path": "rootfs" + }, + "linux": { + "resources": { + "rdmaLimits": { + "mlx5_1": { + "hca_handles": 3, + "hca_objects": 10000 + }, + "mlx4_0": { + "hca_objects": 1000 + }, + "rxe3": { + "hca_objects": 10000 + } + } + } + } +} diff --git a/specs-go/config.go b/specs-go/config.go index 71c9fa773..942649bff 100644 --- a/specs-go/config.go +++ b/specs-go/config.go @@ -320,6 +320,14 @@ type LinuxNetwork struct { Priorities []LinuxInterfacePriority `json:"priorities,omitempty"` } +// LinuxRdmaLimit for Linux cgroup 'rdma' resource management (Linux 4.11) +type LinuxRdmaLimit struct { + // Maximum number of HCA handles that can be opened. Default is "no limit". + HcaHandles *uint32 `json:"hca_handles,omitempty"` + // Maximum number of HCA objects that can be created. Default is "no limit". + HcaObjects *uint32 `json:"hca_objects,omitempty"` +} + // LinuxResources has container runtime resource constraints type LinuxResources struct { // Devices configures the device whitelist. @@ -336,6 +344,10 @@ type LinuxResources struct { HugepageLimits []LinuxHugepageLimit `json:"hugepageLimits,omitempty"` // Network restriction configuration Network *LinuxNetwork `json:"network,omitempty"` + // RdmaLimits resource restriction configuration. + // Limits are a set of key value pairs that define RDMA resource limits, + // where the key is device name and value is resource limits. + RdmaLimits map[string]LinuxRdmaLimit `json:"rdmaLimits,omitempty"` } // LinuxDevice represents the mknod information for a Linux special device file