RFC: libcontainer: add rdmacg support

libcontainer/cgroups/fs/apply_raw.go

@@ -31,6 +31,7 @@ var (
 		&PerfEventGroup{},
 		&FreezerGroup{},
 		&NameGroup{GroupName: "name=systemd", Join: true},
+		&RdmaGroup{},
 	}
 	HugePageSizes, _ = cgroups.GetHugePageSize()
 )

libcontainer/cgroups/fs/rdma.go

@@ -0,0 +1,68 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type RdmaGroup struct {
+}
+
+func (s *RdmaGroup) Name() string {
+	return "rdma"
+}
+
+func (s *RdmaGroup) Apply(d *cgroupData) error {
+	_, err := d.join("rdma")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *RdmaGroup) Set(path string, cgroup *configs.Cgroup) error {
+	for _, rdmaLimit := range cgroup.Resources.RdmaLimits {
+		var handleLimit, objectLimit string
+
+		if rdmaLimit.HcaHandleLimit == -1 {
+			handleLimit = "max"
+		} else {
+			handleLimit = strconv.FormatInt(rdmaLimit.HcaHandleLimit, 10)
+		}
+
+		if rdmaLimit.HcaObjectLimit == -1 {
+			objectLimit = "max"
+		} else {
+			objectLimit = strconv.FormatInt(rdmaLimit.HcaObjectLimit, 10)
+		}
+
+		if err := writeFile(path, "rdma.current", fmt.Sprintf("%s hca_handle=%s hca_object=%s", rdmaLimit.InterfaceName, handleLimit, objectLimit)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *RdmaGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("rdma"))
+}
+
+func (s *RdmaGroup) GetStats(path string, stats *cgroups.Stats) error {
+	str, err := readFile(path, "rdma.current")
+	if err != nil {
+		return err
+	}
+
+	var interfaceName string
+	var rdmaStats cgroups.RdmaStats
+	fmt.Sscanf(str, "%s hca_handle=%d hca_object=%d", interfaceName, rdmaStats.HcaHandle, rdmaStats.HcaObject)
+	stats.RdmaStats[interfaceName] = rdmaStats
+	return nil
+}
+

libcontainer/cgroups/rootless/rootless.go

@@ -28,6 +28,7 @@ var subsystems = []subsystem{
 	&fs.PerfEventGroup{},
 	&fs.FreezerGroup{},
 	&fs.NameGroup{GroupName: "name=systemd"},
+	&fs.RdmaGroup{},
 }
 
 type subsystem interface {

libcontainer/cgroups/stats.go

@@ -92,17 +92,25 @@ type HugetlbStats struct {
 	Failcnt uint64 `json:"failcnt"`
 }
 
+type RdmaStats struct {
+	HcaHandle int64 `json:"hca_handle"`
+	HcaObject int64 `json:"hca_object"`
+}
+
 type Stats struct {
 	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
 	MemoryStats MemoryStats `json:"memory_stats,omitempty"`
 	PidsStats   PidsStats   `json:"pids_stats,omitempty"`
 	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
 	// the map is in the format "size of hugepage: stats of the hugepage"
 	HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
+	// RdmaStats is a map from interface name to its stats
+	RdmaStats map[string]RdmaStats `json:"rmda_stats,omitempty"`
 }
 
 func NewStats() *Stats {
 	memoryStats := MemoryStats{Stats: make(map[string]uint64)}
 	hugetlbStats := make(map[string]HugetlbStats)
-	return &Stats{MemoryStats: memoryStats, HugetlbStats: hugetlbStats}
+	rdmaStats := make(map[string]RdmaStats)
+	return &Stats{MemoryStats: memoryStats, HugetlbStats: hugetlbStats, RdmaStats: rdmaStats}
 }

libcontainer/cgroups/systemd/apply_systemd.go

@@ -61,6 +61,7 @@ var subsystems = subsystemSet{
 	&fs.NetPrioGroup{},
 	&fs.NetClsGroup{},
 	&fs.NameGroup{GroupName: "name=systemd"},
+	&fs.RdmaGroup{},
 }
 
 const (

libcontainer/configs/cgroup_linux.go

@@ -119,4 +119,7 @@ type Resources struct {
 
 	// Set class identifier for container's network packets
 	NetClsClassid uint32 `json:"net_cls_classid_u"`
+
+	// Limit of RDMA related resources
+	RdmaLimits []*RdmaLimit `json:"rdma_limits"`
 }

libcontainer/configs/rdma_limit.go

@@ -0,0 +1,12 @@
+package configs
+
+type RdmaLimit struct {
+	// A name of interface that supports RDMA.
+	InterfaceName string `json:"interface_name"`
+
+	// Limit of HCA Handle. -1 means max.
+	HcaHandleLimit int64 `json:"hca_handle_limit"`
+
+	// Limit of HCA Object. -1 means max.
+	HcaObjectLimit int64 `json:"hca_object_limit"`
+}