From 2e241f7279e4e15a56470213430b5c231c92e7fa Mon Sep 17 00:00:00 2001 From: Parav Pandit Date: Tue, 5 Dec 2017 00:20:15 -0600 Subject: [PATCH] specs-go/config: Define RDMA cgroup Linux kernel 4.11 adds support for RDMA cgroup resource controller. This allows limiting maximum number of open hca_handle and maximum number of hca_objects which can be created by processes. config-linux: Add documentation for Linux RDMA cgroup Add documentation, example and link to kernel documentation for Linux RDMA cgroup. additionalProperties is defined for the JSON Schema draft-04 in [1] with clearer documentation in draft-07 [2]. It is supportd by gojsonschema since xeipuuv/gojsonschema@0572d9d (added additionalProperties with inner schema, 2013-06-21). [1]: https://tools.ietf.org/html/draft-fge-json-schema-validation-00#section-5.4.4 [2]: https://tools.ietf.org/html/draft-handrews-json-schema-validation-00#section-6.5.6 Signed-off-by: Parav Pandit Signed-off-by: W. Trevor King --- config-linux.md | 33 ++++++++++++++++++++++++- schema/config-linux.json | 6 +++++ schema/defs-linux.json | 11 +++++++++ schema/test/config/bad/linux-rdma.json | 15 +++++++++++ schema/test/config/good/linux-rdma.json | 22 +++++++++++++++++ specs-go/config.go | 12 +++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 schema/test/config/bad/linux-rdma.json create mode 100644 schema/test/config/good/linux-rdma.json diff --git a/config-linux.md b/config-linux.md index f73d893c9..e33cea5b7 100644 --- a/config-linux.md +++ b/config-linux.md @@ -169,7 +169,7 @@ In addition to any devices configured with this setting, the runtime MUST also s ## Control groups Also known as cgroups, they are used to restrict resource usage for a container and handle device access. -cgroups provide controls (through controllers) to restrict cpu, memory, IO, pids and network for the container. +cgroups provide controls (through controllers) to restrict cpu, memory, IO, pids, network and RDMA resources for the container. For more information, see the [kernel cgroups documentation][cgroup-v1]. ### Cgroups Path @@ -455,6 +455,36 @@ The following parameters can be specified to set up the controller: } ``` +### RDMA + +**`rdma`** (object, OPTIONAL) represents the cgroup subsystem `rdma`. +For more information, see the kernel cgroups documentation about [rdma][cgroup-v1-rdma]. + +The name of the device to limit is the entry key. +Entry values are objects with the following properties: + +* **`hcaHandles`** *(uint32, OPTIONAL)* - specifies the maximum number of hca_handles in the cgroup +* **`hcaObjects`** *(uint32, OPTIONAL)* - specifies the maximum number of hca_objects in the cgroup + +You MUST specify at least one of the `hcaHandles` or `hcaObjects` in a given entry, and MAY specify both. + +#### Example + +```json +"rdma": { + "mlx5_1": { + "hcaHandles": 3, + "hcaObjects": 10000 + }, + "mlx4_0": { + "hcaObjects": 1000 + }, + "rxe3": { + "hcaObjects": 10000 + } +} +``` + ## IntelRdt **`intelRdt`** (object, OPTIONAL) represents the [Intel Resource Director Technology][intel-rdt-cat-kernel-interface]. @@ -647,6 +677,7 @@ The following parameters can be specified to set up seccomp: [cgroup-v1-net-cls]: https://www.kernel.org/doc/Documentation/cgroup-v1/net_cls.txt [cgroup-v1-net-prio]: https://www.kernel.org/doc/Documentation/cgroup-v1/net_prio.txt [cgroup-v1-pids]: https://www.kernel.org/doc/Documentation/cgroup-v1/pids.txt +[cgroup-v1-rdma]: https://www.kernel.org/doc/Documentation/cgroup-v1/rdma.txt [cgroup-v2]: https://www.kernel.org/doc/Documentation/cgroup-v2.txt [devices]: https://www.kernel.org/doc/Documentation/admin-guide/devices.txt [devpts]: https://www.kernel.org/doc/Documentation/filesystems/devpts.txt diff --git a/schema/config-linux.json b/schema/config-linux.json index 85870f06f..5a3fd50e1 100644 --- a/schema/config-linux.json +++ b/schema/config-linux.json @@ -175,6 +175,12 @@ } } } + }, + "rdma": { + "type": "object", + "additionalProperties": { + "$ref": "defs-linux.json#/definitions/Rdma" + } } } }, diff --git a/schema/defs-linux.json b/schema/defs-linux.json index 4d9620a4a..0735e6b89 100644 --- a/schema/defs-linux.json +++ b/schema/defs-linux.json @@ -240,6 +240,17 @@ "priority" ] }, + "Rdma": { + "type": "object", + "properties": { + "hcaHandles": { + "$ref": "defs.json#/definitions/uint32" + }, + "hcaObjects": { + "$ref": "defs.json#/definitions/uint32" + } + } + }, "NamespaceType": { "type": "string", "enum": [ diff --git a/schema/test/config/bad/linux-rdma.json b/schema/test/config/bad/linux-rdma.json new file mode 100644 index 000000000..5a7ac695d --- /dev/null +++ b/schema/test/config/bad/linux-rdma.json @@ -0,0 +1,15 @@ +{ + "ociVersion": "1.0.0", + "root": { + "path": "rootfs" + }, + "linux": { + "resources": { + "rdma": { + "mlx5_1": { + "hcaHandles": "not a uint32" + } + } + } + } +} diff --git a/schema/test/config/good/linux-rdma.json b/schema/test/config/good/linux-rdma.json new file mode 100644 index 000000000..e70ed5320 --- /dev/null +++ b/schema/test/config/good/linux-rdma.json @@ -0,0 +1,22 @@ +{ + "ociVersion": "1.0.0", + "root": { + "path": "rootfs" + }, + "linux": { + "resources": { + "rdma": { + "mlx5_1": { + "hcaHandles": 3, + "hcaObjects": 10000 + }, + "mlx4_0": { + "hcaObjects": 1000 + }, + "rxe3": { + "hcaObjects": 10000 + } + } + } + } +} diff --git a/specs-go/config.go b/specs-go/config.go index 71c9fa773..841eacb2d 100644 --- a/specs-go/config.go +++ b/specs-go/config.go @@ -320,6 +320,14 @@ type LinuxNetwork struct { Priorities []LinuxInterfacePriority `json:"priorities,omitempty"` } +// LinuxRdma for Linux cgroup 'rdma' resource management (Linux 4.11) +type LinuxRdma struct { + // Maximum number of HCA handles that can be opened. Default is "no limit". + HcaHandles *uint32 `json:"hcaHandles,omitempty"` + // Maximum number of HCA objects that can be created. Default is "no limit". + HcaObjects *uint32 `json:"hcaObjects,omitempty"` +} + // LinuxResources has container runtime resource constraints type LinuxResources struct { // Devices configures the device whitelist. @@ -336,6 +344,10 @@ type LinuxResources struct { HugepageLimits []LinuxHugepageLimit `json:"hugepageLimits,omitempty"` // Network restriction configuration Network *LinuxNetwork `json:"network,omitempty"` + // Rdma resource restriction configuration. + // Limits are a set of key value pairs that define RDMA resource limits, + // where the key is device name and value is resource limits. + Rdma map[string]LinuxRdma `json:"rdma,omitempty"` } // LinuxDevice represents the mknod information for a Linux special device file