From c19a1648d21c694e201d3cfa7aa155dcc034562a Mon Sep 17 00:00:00 2001 From: amaslennikov Date: Mon, 31 Jul 2023 18:32:54 +0300 Subject: [PATCH] Add support for CDI https://github.com/container-orchestrated-devices/container-device-interface Add copyrights to all edited files Switch to kustomize for deployment artifacts Update Readme Signed-off-by: amaslennikov --- README.md | 21 +-- cmd/k8s-rdma-shared-dp/main.go | 27 +++- .../k8s/base/configmap.yaml | 0 .../k8s/base/daemonset.yaml | 0 deployment/k8s/base/kustomization.yaml | 6 + deployment/k8s/overlay/cdi.yaml | 24 +++ deployment/k8s/overlay/kustomization.yaml | 8 + go.mod | 10 ++ go.sum | 74 ++++++++- ...8s-rdma-shared-dev-plugin-ds-pre-1.16.yaml | 50 ------ pkg/cdi/cdi.go | 144 ++++++++++++++++++ pkg/cdi/cdi_test.go | 90 +++++++++++ pkg/cdi/mocks/cdi.go | 68 +++++++++ pkg/resources/resources_manager.go | 43 +++++- pkg/resources/resources_manager_test.go | 36 ++++- pkg/resources/server.go | 102 ++++++++++--- pkg/resources/server_test.go | 80 ++++++++-- pkg/types/types.go | 18 +++ 18 files changed, 703 insertions(+), 98 deletions(-) rename images/k8s-rdma-shared-dev-plugin-config-map.yaml => deployment/k8s/base/configmap.yaml (100%) rename images/k8s-rdma-shared-dev-plugin-ds.yaml => deployment/k8s/base/daemonset.yaml (100%) create mode 100644 deployment/k8s/base/kustomization.yaml create mode 100644 deployment/k8s/overlay/cdi.yaml create mode 100644 deployment/k8s/overlay/kustomization.yaml delete mode 100644 images/k8s-rdma-shared-dev-plugin-ds-pre-1.16.yaml create mode 100644 pkg/cdi/cdi.go create mode 100644 pkg/cdi/cdi_test.go create mode 100644 pkg/cdi/mocks/cdi.go diff --git a/README.md b/README.md index 38e9a3f..5d855d6 100644 --- a/README.md +++ b/README.md @@ -17,21 +17,16 @@ available at mellanox/k8s-rdma-shared-dev-plugin. Make sure to configure ib0 or appropriate IPoIB netdevice as the parent netdevice for creating overlay/virtual netdevices. -**2.** Create ConfigMap +**2.** Create ConfigMap and deploy Device Plugin -Create config map to describe mode as "hca" mode. This is per node configuration. +Deploy device plugin and create config map to describe mode as "hca" mode. This is per node configuration. ``` -kubectl create -f images/k8s-rdma-shared-dev-plugin-config-map.yaml +cd deployment/k8s/base +kubectl apply -k . ``` -**3.** Deploy device plugin - -``` -kubectl create -f images/k8s-rdma-shared-dev-plugin-ds.yaml -``` - -**4.** Create Test pod +**3.** Create Test pod Create test pod which requests 1 vhca resource. @@ -39,6 +34,12 @@ Create test pod which requests 1 vhca resource. kubectl create -f example/test-hca-pod.yaml ``` +### Deploy the device plugin with CDI support +To use the device plugin with [CDI](https://github.com/cncf-tags/container-device-interface) support, do the following: +``` +cd deployment/k8s/base/overlay +kubectl apply -k . +``` # How to use device plugin for RDMA The device plugin can be used with macvlan for RDMA, to do the following steps: diff --git a/cmd/k8s-rdma-shared-dp/main.go b/cmd/k8s-rdma-shared-dp/main.go index 92ef394..762fb33 100644 --- a/cmd/k8s-rdma-shared-dp/main.go +++ b/cmd/k8s-rdma-shared-dp/main.go @@ -1,3 +1,21 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + package main import ( @@ -31,15 +49,22 @@ func main() { flag.BoolVar(&versionOpt, "v", false, "Show application version") flag.StringVar( &configFilePath, "config-file", resources.DefaultConfigFilePath, "path to device plugin config file") + useCdi := false + flag.BoolVar(&useCdi, "use-cdi", false, + "Use Container Device Interface to expose devices in containers") flag.Parse() if versionOpt { fmt.Printf("%s\n", printVersionString()) return } + if useCdi { + log.Println("CDI enabled") + } + log.Println("Starting K8s RDMA Shared Device Plugin version=", version) - rm := resources.NewResourceManager(configFilePath) + rm := resources.NewResourceManager(configFilePath, useCdi) log.Println("resource manager reading configs") if err := rm.ReadConfig(); err != nil { diff --git a/images/k8s-rdma-shared-dev-plugin-config-map.yaml b/deployment/k8s/base/configmap.yaml similarity index 100% rename from images/k8s-rdma-shared-dev-plugin-config-map.yaml rename to deployment/k8s/base/configmap.yaml diff --git a/images/k8s-rdma-shared-dev-plugin-ds.yaml b/deployment/k8s/base/daemonset.yaml similarity index 100% rename from images/k8s-rdma-shared-dev-plugin-ds.yaml rename to deployment/k8s/base/daemonset.yaml diff --git a/deployment/k8s/base/kustomization.yaml b/deployment/k8s/base/kustomization.yaml new file mode 100644 index 0000000..1d14d1d --- /dev/null +++ b/deployment/k8s/base/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - configmap.yaml + - daemonset.yaml diff --git a/deployment/k8s/overlay/cdi.yaml b/deployment/k8s/overlay/cdi.yaml new file mode 100644 index 0000000..c447fd0 --- /dev/null +++ b/deployment/k8s/overlay/cdi.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: rdma-shared-dp-ds + namespace: kube-system +spec: + template: + spec: + containers: + - name: k8s-rdma-shared-dp-ds + volumeMounts: + - name: default-cdi + mountPath: /etc/cdi/ + - name: dynamic-cdi + mountPath: /var/run/cdi + volumes: + - name: default-cdi + hostPath: + path: /etc/cdi + type: DirectoryOrCreate + - name: dynamic-cdi + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate diff --git a/deployment/k8s/overlay/kustomization.yaml b/deployment/k8s/overlay/kustomization.yaml new file mode 100644 index 0000000..af90021 --- /dev/null +++ b/deployment/k8s/overlay/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../base + +patchesStrategicMerge: + - cdi.yaml \ No newline at end of file diff --git a/go.mod b/go.mod index 8e8aa16..4ee0d1d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/Mellanox/rdmamap v1.1.0 + github.com/container-orchestrated-devices/container-device-interface v0.5.4 github.com/jaypipes/ghw v0.6.1 github.com/jaypipes/pcidb v1.0.0 github.com/onsi/ginkgo v1.11.0 @@ -19,16 +20,24 @@ require ( require ( github.com/StackExchange/wmi v1.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/hpcloud/tail v1.0.0 // indirect + github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/opencontainers/runc v1.1.2 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect + github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect + github.com/opencontainers/selinux v1.10.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.0 // indirect + github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vishvananda/netns v0.0.4 // indirect + golang.org/x/mod v0.8.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect @@ -39,6 +48,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect howett.net/plist v1.0.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index fb7d3ce..d87f58f 100644 --- a/go.sum +++ b/go.sum @@ -5,19 +5,36 @@ github.com/Mellanox/rdmamap v1.1.0/go.mod h1:fN+/V9lf10ABnDCwTaXRjeeWijLt2iVLETn github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= +github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= +github.com/container-orchestrated-devices/container-device-interface v0.6.1 h1:mz77uJoP8im/4Zins+mPqt677ZMaflhoGaYrRAl5jvA= +github.com/container-orchestrated-devices/container-device-interface v0.6.1/go.mod h1:40T6oW59rFrL/ksiSs7q45GzjGlbvxnA4xaK6cyq+kA= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -27,10 +44,16 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/go-genproto v0.0.0-20200117163144-32f20d992d24 h1:iqmAazjqJalhqqPoTsjnqI2wPlgVbQJ6JNdHsbhA6FA= github.com/googleapis/go-genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -42,8 +65,9 @@ github.com/jaypipes/pcidb v1.0.0/go.mod h1:TnYUvqhPBzCKnH34KrIX22kAeEbDCSRJ9cqLR github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -51,31 +75,62 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw= +github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0= +github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= +github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -86,6 +141,8 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -94,6 +151,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -108,9 +166,18 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= @@ -134,6 +201,7 @@ google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -158,3 +226,5 @@ howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= k8s.io/kubelet v0.27.3 h1:5WhTV1iiBu9q/rr+gvy65LQ+K/e7dmgcaYjys5ipLqY= k8s.io/kubelet v0.27.3/go.mod h1:Mz42qgZZgWgPmOJEYaR5evmh+EoSwFzEvPBozA2y9mg= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/images/k8s-rdma-shared-dev-plugin-ds-pre-1.16.yaml b/images/k8s-rdma-shared-dev-plugin-ds-pre-1.16.yaml deleted file mode 100644 index cf2b23f..0000000 --- a/images/k8s-rdma-shared-dev-plugin-ds-pre-1.16.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: rdma-shared-dp-ds - namespace: kube-system -spec: - template: - metadata: - # Mark this pod as a critical add-on; when enabled, the critical add-on scheduler - # reserves resources for critical add-on pods so that they can be rescheduled after - # a failure. This annotation works in tandem with the toleration below. - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - name: rdma-shared-dp-ds - spec: - hostNetwork: true - tolerations: - # Allow this pod to be rescheduled while the node is in "critical add-ons only" mode. - # This, along with the annotation above marks this pod as a critical add-on. - - key: CriticalAddonsOnly - operator: Exists - containers: - - image: mellanox/k8s-rdma-shared-dev-plugin - name: k8s-rdma-shared-dp-ds - imagePullPolicy: IfNotPresent - securityContext: - privileged: true - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - readOnly: false - - name: plugins-registry - mountPath: /var/lib/kubelet/plugins_registry - readOnly: false - - name: config - mountPath: /k8s-rdma-shared-dev-plugin - volumes: - - name: device-plugin - hostPath: - path: /var/lib/kubelet/device-plugins - - name: plugins-registry - hostPath: - path: /var/lib/kubelet/plugins_registry - - name: config - configMap: - name: rdma-devices - items: - - key: config.json - path: config.json diff --git a/pkg/cdi/cdi.go b/pkg/cdi/cdi.go new file mode 100644 index 0000000..9e6a5b2 --- /dev/null +++ b/pkg/cdi/cdi.go @@ -0,0 +1,144 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + +package cdi + +import ( + "errors" + "log" + "os" + "path/filepath" + + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + cdiSpecs "github.com/container-orchestrated-devices/container-device-interface/specs-go" + + "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types" +) + +// CDI represents CDI API required by Device plugin +type CDI interface { + CreateCDISpec(resourcePrefix, resourceName, poolName string, devices []types.PciNetDevice) error + CreateContainerAnnotations( + devices []types.PciNetDevice, resourcePrefix, resourceKind string) (map[string]string, error) +} + +// impl implements CDI interface +type impl struct { +} + +// CreateCDISpec creates CDI spec file with specified devices +func (c *impl) CreateCDISpec( + resourcePrefix, resourceName, poolName string, devices []types.PciNetDevice) error { + log.Printf("creating CDI spec for \"%s\" resource", resourceName) + + cdiDevices := make([]cdiSpecs.Device, 0) + cdiSpec := cdiSpecs.Spec{ + Version: cdiSpecs.CurrentVersion, + Kind: resourcePrefix + "/" + resourceName, + Devices: cdiDevices, + } + + for _, dev := range devices { + containerEdit := cdiSpecs.ContainerEdits{ + DeviceNodes: make([]*cdiSpecs.DeviceNode, 0), + } + + rdmaSpec := dev.GetRdmaSpec() + for _, spec := range rdmaSpec { + deviceNode := cdiSpecs.DeviceNode{ + Path: spec.ContainerPath, + HostPath: spec.HostPath, + Permissions: "rw", + } + containerEdit.DeviceNodes = append(containerEdit.DeviceNodes, &deviceNode) + } + + device := cdiSpecs.Device{ + Name: dev.GetPciAddr(), + ContainerEdits: containerEdit, + } + cdiSpec.Devices = append(cdiSpec.Devices, device) + } + + err := cdi.GetRegistry().SpecDB().WriteSpec(&cdiSpec, resourcePrefix+"_"+poolName) + if err != nil { + log.Printf("createCDISpec(): can not create CDI json: %q", err) + return err + } + + log.Printf("createCDISpec(): listing cache") + for _, vendor := range cdi.GetRegistry().SpecDB().ListVendors() { + for _, spec := range cdi.GetRegistry().SpecDB().GetVendorSpecs(vendor) { + for _, dev := range spec.Devices { + log.Printf("createCDISpec(): device: %s, %s, %s", vendor, spec.Kind, dev.Name) + } + } + } + + return nil +} + +// CreateContainerAnnotations creates container annotations based on CDI spec for a container runtime +func (c *impl) CreateContainerAnnotations( + devices []types.PciNetDevice, resourceNamePrefix, resourceKind string) (map[string]string, error) { + if len(devices) == 0 { + return nil, errors.New("devices list is empty") + } + annotations := make(map[string]string, 0) + annoKey, err := cdi.AnnotationKey(resourceNamePrefix, resourceKind) + if err != nil { + log.Printf("can not create container annotation: %q", err) + return nil, err + } + deviceNames := make([]string, len(devices)) + for i, dev := range devices { + deviceNames[i] = cdi.QualifiedName(resourceNamePrefix, resourceKind, dev.GetPciAddr()) + } + annoValue, err := cdi.AnnotationValue(deviceNames) + if err != nil { + log.Printf("can not create container annotation: %q", err) + return nil, err + } + annotations[annoKey] = annoValue + + log.Printf("created CDI annotations: %v", annotations) + + return annotations, nil +} + +// CleanupSpecs removes previously-created CDI specs +func CleanupSpecs(specFilePrefix string) error { + for _, dir := range cdi.GetRegistry().GetSpecDirectories() { + specs, err := filepath.Glob(filepath.Join(dir, specFilePrefix+"*")) + if err != nil { + return err + } + for _, spec := range specs { + log.Printf("Cleaning up CDI spec file: %s", spec) + if err := os.Remove(spec); err != nil { + return err + } + } + } + + return nil +} + +func New() CDI { + return &impl{} +} diff --git a/pkg/cdi/cdi_test.go b/pkg/cdi/cdi_test.go new file mode 100644 index 0000000..c80d5b0 --- /dev/null +++ b/pkg/cdi/cdi_test.go @@ -0,0 +1,90 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + +package cdi_test + +import ( + "os" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + cdiLib "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + + "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/cdi" + "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types" + "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types/mocks" + "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/utils" +) + +func TestCdi(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CDI Suite") +} + +var _ = Describe("Cdi", func() { + Context("successfully create CDI spec file", func() { + It("should create CDI spec file", func() { + staticDir := "cdi_static" + dynamicDir := "cdi_dynamic" + fs := &utils.FakeFilesystem{ + Dirs: []string{staticDir, dynamicDir}, + } + defer fs.Use()() + cdiLib.GetRegistry(cdiLib.WithSpecDirs(fs.RootDir+"/"+staticDir, fs.RootDir+"/"+dynamicDir)) + + device := mocks.PciNetDevice{} + device.On("GetPciAddr").Return("0000:00:00.1") + device.On("GetRdmaSpec").Return([]*pluginapi.DeviceSpec{ + {HostPath: "host_path", ContainerPath: "container_path"}, + }) + + err := cdi.New().CreateCDISpec("test-prefix", "test-name", "test-pool", []types.PciNetDevice{&device}) + Expect(err).NotTo(HaveOccurred()) + + cdiSpec, err := os.ReadFile(fs.RootDir + "/" + dynamicDir + "/test-prefix_test-pool.yaml") + Expect(err).NotTo(HaveOccurred()) + + substring := ` +devices: +- containerEdits: + deviceNodes: + - hostPath: host_path + path: container_path + permissions: rw + name: "0000:00:00.1" +kind: test-prefix/test-name` + Expect(string(cdiSpec)).To(ContainSubstring(substring)) + }) + }) + Context("successfully create container annotation", func() { + It("should return container annotation", func() { + device := mocks.PciNetDevice{} + device.On("GetPciAddr").Return("0000:00:00.1") + cdi := cdi.New() + annotations, err := cdi.CreateContainerAnnotations([]types.PciNetDevice{&device}, "example.com", "net") + Expect(err).NotTo(HaveOccurred()) + Expect(len(annotations)).To(Equal(1)) + annoKey := "cdi.k8s.io/example.com_net" + annoVal := "example.com/net=0000:00:00.1" + Expect(annotations[annoKey]).To(Equal(annoVal)) + }) + }) +}) diff --git a/pkg/cdi/mocks/cdi.go b/pkg/cdi/mocks/cdi.go new file mode 100644 index 0000000..c41d336 --- /dev/null +++ b/pkg/cdi/mocks/cdi.go @@ -0,0 +1,68 @@ +// Code generated by mockery v2.26.1. DO NOT EDIT. + +package mocks + +import ( + types "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types" + mock "github.com/stretchr/testify/mock" +) + +// CDI is an autogenerated mock type for the CDI type +type CDI struct { + mock.Mock +} + +// CreateCDISpec provides a mock function with given fields: resourcePrefix, resourceName, poolName, devices +func (_m *CDI) CreateCDISpec(resourcePrefix string, resourceName string, poolName string, devices []types.PciNetDevice) error { + ret := _m.Called(resourcePrefix, resourceName, poolName, devices) + + var r0 error + if rf, ok := ret.Get(0).(func(string, string, string, []types.PciNetDevice) error); ok { + r0 = rf(resourcePrefix, resourceName, poolName, devices) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// CreateContainerAnnotations provides a mock function with given fields: devices, resourcePrefix, resourceKind +func (_m *CDI) CreateContainerAnnotations(devices []types.PciNetDevice, resourcePrefix string, resourceKind string) (map[string]string, error) { + ret := _m.Called(devices, resourcePrefix, resourceKind) + + var r0 map[string]string + var r1 error + if rf, ok := ret.Get(0).(func([]types.PciNetDevice, string, string) (map[string]string, error)); ok { + return rf(devices, resourcePrefix, resourceKind) + } + if rf, ok := ret.Get(0).(func([]types.PciNetDevice, string, string) map[string]string); ok { + r0 = rf(devices, resourcePrefix, resourceKind) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]string) + } + } + + if rf, ok := ret.Get(1).(func([]types.PciNetDevice, string, string) error); ok { + r1 = rf(devices, resourcePrefix, resourceKind) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type mockConstructorTestingTNewCDI interface { + mock.TestingT + Cleanup(func()) +} + +// NewCDI creates a new instance of CDI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewCDI(t mockConstructorTestingTNewCDI) *CDI { + mock := &CDI{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/resources/resources_manager.go b/pkg/resources/resources_manager.go index 747da62..311abdf 100644 --- a/pkg/resources/resources_manager.go +++ b/pkg/resources/resources_manager.go @@ -1,3 +1,21 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + package resources import ( @@ -12,6 +30,7 @@ import ( "github.com/jaypipes/ghw" "github.com/vishvananda/netlink" + "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/cdi" "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types" "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/utils" ) @@ -52,9 +71,10 @@ type resourceManager struct { netlinkManager types.NetlinkManager rds types.RdmaDeviceSpec PeriodicUpdateInterval time.Duration + useCdi bool } -func NewResourceManager(configFile string) types.ResourceManager { +func NewResourceManager(configFile string, useCdi bool) types.ResourceManager { watcherMode := detectPluginWatchMode(activeSockDir) if watcherMode { fmt.Println("Using Kubelet Plugin Registry Mode") @@ -68,6 +88,7 @@ func NewResourceManager(configFile string) types.ResourceManager { watchMode: watcherMode, netlinkManager: &netlinkManager{}, rds: NewRdmaDeviceSpec(requiredRdmaDevices), + useCdi: useCdi, } } @@ -201,7 +222,13 @@ func (rm *resourceManager) InitServers() error { log.Printf("Warning: no devices in device pool, creating empty resource server for %s", config.ResourceName) } - rs, err := newResourceServer(config, filteredDevices, rm.watchMode, rm.socketSuffix) + if rm.useCdi { + err := cdi.CleanupSpecs(cdiResourcePrefix) + if err != nil { + return err + } + } + rs, err := newResourceServer(config, filteredDevices, rm.watchMode, rm.socketSuffix, rm.useCdi) if err != nil { return err } @@ -227,6 +254,12 @@ func (rm *resourceManager) StartAllServers() error { // StopAllServers stop all servers func (rm *resourceManager) StopAllServers() error { + if rm.useCdi { + err := cdi.CleanupSpecs(cdiResourcePrefix) + if err != nil { + return err + } + } for _, rs := range rm.resourceServers { if err := rs.Stop(); err != nil { return err @@ -237,6 +270,12 @@ func (rm *resourceManager) StopAllServers() error { // RestartAllServers restart all servers func (rm *resourceManager) RestartAllServers() error { + if rm.useCdi { + err := cdi.CleanupSpecs(cdiResourcePrefix) + if err != nil { + return err + } + } for _, rs := range rm.resourceServers { if err := rs.Restart(); err != nil { return err diff --git a/pkg/resources/resources_manager_test.go b/pkg/resources/resources_manager_test.go index 57b52de..98b7c92 100644 --- a/pkg/resources/resources_manager_test.go +++ b/pkg/resources/resources_manager_test.go @@ -1,3 +1,21 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + package resources import ( @@ -46,7 +64,7 @@ var _ = Describe("ResourcesManger", func() { activeSockDir = activeSockDirBackUP }() - obj := NewResourceManager(DefaultConfigFilePath) + obj := NewResourceManager(DefaultConfigFilePath, false) rm := obj.(*resourceManager) Expect(rm.watchMode).To(Equal(true)) }) @@ -58,7 +76,7 @@ var _ = Describe("ResourcesManger", func() { activeSockDir = activeSockDirBackUP }() - obj := NewResourceManager(DefaultConfigFilePath) + obj := NewResourceManager(DefaultConfigFilePath, false) rm := obj.(*resourceManager) Expect(rm.watchMode).To(Equal(false)) }) @@ -438,6 +456,20 @@ var _ = Describe("ResourcesManger", func() { err := rm.InitServers() Expect(err).To(HaveOccurred()) }) + It("Init server with CDI support", func() { + var configlist []*types.UserConfig + rm := &resourceManager{useCdi: true} + + configlist = append(configlist, &types.UserConfig{ + ResourceName: "test_config", + ResourcePrefix: "test_prefix", + RdmaHcaMax: 100, + Devices: []string{"ib0"}}) + + rm.configList = configlist + err := rm.InitServers() + Expect(err).ToNot(HaveOccurred()) + }) }) Context("StartAllServers", func() { It("start valid server with watcher enabled", func() { diff --git a/pkg/resources/server.go b/pkg/resources/server.go index 9f2b774..8d7dc11 100644 --- a/pkg/resources/server.go +++ b/pkg/resources/server.go @@ -1,3 +1,21 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + package resources import ( @@ -16,13 +34,16 @@ import ( pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" registerapi "k8s.io/kubelet/pkg/apis/pluginregistration/v1" + "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/cdi" "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types" ) const ( // Local use - cDialTimeout = 5 * time.Second - watchWaitTime = 5 * time.Second + cDialTimeout = 5 * time.Second + watchWaitTime = 5 * time.Second + cdiResourcePrefix = "nvidia.com" + cdiResourceKind = "net-rdma" ) type resourcesServerPort struct { @@ -40,9 +61,13 @@ type resourceServer struct { rsConnector types.ResourceServerPort rdmaHcaMax int // Mutex protects devs and deviceSpec - mutex sync.RWMutex - devs []*pluginapi.Device - deviceSpec []*pluginapi.DeviceSpec + mutex sync.RWMutex + devs []*pluginapi.Device + deviceSpec []*pluginapi.DeviceSpec + pciDevices []types.PciNetDevice + useCdi bool + cdi cdi.CDI + cdiResourceName string } func (rsc *resourcesServerPort) GetServer() *grpc.Server { @@ -98,7 +123,7 @@ func (rsc *resourcesServerPort) Dial(unixSocketPath string, timeout time.Duratio // newResourceServer returns an initialized server func newResourceServer(config *types.UserConfig, devices []types.PciNetDevice, watcherMode bool, - socketSuffix string) (types.ResourceServer, error) { + socketSuffix string, useCdi bool) (types.ResourceServer, error) { var devs []*pluginapi.Device sockDir := activeSockDir @@ -132,17 +157,21 @@ func newResourceServer(config *types.UserConfig, devices []types.PciNetDevice, w socketName := fmt.Sprintf("%s.%s", config.ResourceName, socketSuffix) return &resourceServer{ - resourceName: fmt.Sprintf("%s/%s", config.ResourcePrefix, config.ResourceName), - socketName: socketName, - socketPath: filepath.Join(sockDir, socketName), - watchMode: watcherMode, - devs: devs, - deviceSpec: deviceSpec, - stopWatcher: make(chan bool), - updateResource: make(chan bool, 1), - health: make(chan *pluginapi.Device), - rsConnector: &resourcesServerPort{}, - rdmaHcaMax: config.RdmaHcaMax, + resourceName: fmt.Sprintf("%s/%s", config.ResourcePrefix, config.ResourceName), + socketName: socketName, + socketPath: filepath.Join(sockDir, socketName), + watchMode: watcherMode, + devs: devs, + deviceSpec: deviceSpec, + stopWatcher: make(chan bool), + updateResource: make(chan bool, 1), + health: make(chan *pluginapi.Device), + rsConnector: &resourcesServerPort{}, + rdmaHcaMax: config.RdmaHcaMax, + pciDevices: devices, + useCdi: useCdi, + cdi: cdi.New(), + cdiResourceName: config.ResourceName, }, nil } @@ -275,6 +304,14 @@ func (rs *resourceServer) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlu return err } + rs.mutex.RLock() + err := rs.updateCDISpec() + rs.mutex.RUnlock() + if err != nil { + log.Printf("cannot update CDI specs: %v", err) + return err + } + for { select { case <-s.Context().Done(): @@ -291,10 +328,28 @@ func (rs *resourceServer) ListAndWatch(e *pluginapi.Empty, s pluginapi.DevicePlu rs.updateResource <- true return err } + err := rs.updateCDISpec() + if err != nil { + log.Printf("cannot update CDI specs: %v", err) + return err + } } } } +func (rs *resourceServer) updateCDISpec() error { + // check if CDI mode is enabled + if !rs.useCdi { + return nil + } + err := rs.cdi.CreateCDISpec(cdiResourcePrefix, cdiResourceKind, rs.cdiResourceName, rs.pciDevices) + if err != nil { + log.Printf("updateCDISpec(): error creating CDI spec: %v", err) + return err + } + return nil +} + func (rs *resourceServer) sendDevices(resp *pluginapi.ListAndWatchResponse, s pluginapi.DevicePlugin_ListAndWatchServer) error { rs.mutex.RLock() @@ -320,8 +375,17 @@ func (rs *resourceServer) Allocate(ctx context.Context, r *pluginapi.AllocateReq ress := make([]*pluginapi.ContainerAllocateResponse, len(r.GetContainerRequests())) for i := range r.GetContainerRequests() { - ress[i] = &pluginapi.ContainerAllocateResponse{ - Devices: rs.deviceSpec, + ress[i] = &pluginapi.ContainerAllocateResponse{} + + if rs.useCdi { + var err error + ress[i].Annotations, err = rs.cdi.CreateContainerAnnotations( + rs.pciDevices, cdiResourcePrefix, cdiResourceKind) + if err != nil { + return nil, fmt.Errorf("cant create container annotation: %s", err) + } + } else { + ress[i].Devices = rs.deviceSpec } } diff --git a/pkg/resources/server_test.go b/pkg/resources/server_test.go index f31b690..81b5a93 100644 --- a/pkg/resources/server_test.go +++ b/pkg/resources/server_test.go @@ -1,3 +1,21 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + package resources import ( @@ -8,6 +26,7 @@ import ( "sync" "time" + cdiMocks "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/cdi/mocks" "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types" "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/types/mocks" "github.com/Mellanox/k8s-rdma-shared-dev-plugin/pkg/utils" @@ -57,7 +76,7 @@ var _ = Describe("resourceServer tests", func() { } defer fs.Use()() conf := &types.UserConfig{ResourceName: "test_server", ResourcePrefix: "rdma", RdmaHcaMax: 100} - obj, err := newResourceServer(conf, fakeDeviceList, true, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, true, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) Expect(rs.resourceName).To(Equal("rdma/test_server")) @@ -72,7 +91,7 @@ var _ = Describe("resourceServer tests", func() { } defer fs.Use()() conf := &types.UserConfig{ResourceName: "test_server", ResourcePrefix: "rdma", RdmaHcaMax: 0} - obj, err := newResourceServer(conf, fakeDeviceList, true, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, true, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) Expect(rs.resourceName).To(Equal("rdma/test_server")) @@ -91,7 +110,7 @@ var _ = Describe("resourceServer tests", func() { fakePciDevice.On("GetRdmaSpec").Return([]*pluginapi.DeviceSpec{}) fakePciDevice.On("GetPciAddr").Return("0000:02:00.0") deviceList := []types.PciNetDevice{fakePciDevice} - obj, err := newResourceServer(conf, deviceList, true, "socket") + obj, err := newResourceServer(conf, deviceList, true, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) Expect(rs.resourceName).To(Equal("rdma/test_server")) @@ -106,7 +125,7 @@ var _ = Describe("resourceServer tests", func() { } defer fs.Use()() conf := &types.UserConfig{ResourceName: "test_server", ResourcePrefix: "rdma", RdmaHcaMax: 100} - obj, err := newResourceServer(conf, fakeDeviceList, false, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, false, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) Expect(rs.resourceName).To(Equal("rdma/test_server")) @@ -121,7 +140,7 @@ var _ = Describe("resourceServer tests", func() { } defer fs.Use()() conf := &types.UserConfig{ResourceName: "test_server", ResourcePrefix: "rdma", RdmaHcaMax: 0} - obj, err := newResourceServer(conf, fakeDeviceList, false, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, false, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) Expect(rs.resourceName).To(Equal("rdma/test_server")) @@ -131,7 +150,7 @@ var _ = Describe("resourceServer tests", func() { }) It("server with plugin with invalid max number of resources", func() { conf := &types.UserConfig{ResourceName: "test_server", ResourcePrefix: "rdma", RdmaHcaMax: -100} - obj, err := newResourceServer(conf, fakeDeviceList, true, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, true, "socket", false) Expect(err).To(HaveOccurred()) Expect(obj).To(BeNil()) }) @@ -364,7 +383,7 @@ var _ = Describe("resourceServer tests", func() { } defer fs.Use()() conf := &types.UserConfig{RdmaHcaMax: 100, ResourcePrefix: "rdma", ResourceName: "fake"} - obj, err := newResourceServer(conf, fakeDeviceList, true, "fake") + obj, err := newResourceServer(conf, fakeDeviceList, true, "fake", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) @@ -388,6 +407,31 @@ var _ = Describe("resourceServer tests", func() { Expect(len(s.devices)).To(Equal(100)) Expect(s.devices[5].Health).To(Equal(pluginapi.Unhealthy)) }) + It("Ensure that CDI spec is updated on ListAndWatch call", func() { + fs := utils.FakeFilesystem{ + Dirs: []string{fakeNetDevicePath}, + Symlinks: map[string]string{path.Join(fakeNetDevicePath, "device"): "../../../0000:02:00.0"}, + } + defer fs.Use()() + conf := &types.UserConfig{RdmaHcaMax: 1, ResourcePrefix: "rdma", ResourceName: "fake"} + obj, err := newResourceServer(conf, fakeDeviceList, true, "fake", true) + Expect(err).ToNot(HaveOccurred()) + + cdi := &cdiMocks.CDI{} + cdi.On("CreateCDISpec", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + rs := obj.(*resourceServer) + rs.cdi = cdi + + ctx, cancel := context.WithCancel(context.Background()) + s := &devPluginListAndWatchServerMock{} + s.SetContext(ctx) + + cancel() + + err = rs.ListAndWatch(nil, s) + cdi.AssertCalled(GinkgoT(), "CreateCDISpec", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + Expect(err).ToNot(HaveOccurred()) + }) It("Stop ListAndWatch when stream is closed", func() { rs := resourceServer{resourceName: "fake", socketName: "fake.sock"} s := &devPluginListAndWatchServerMock{} @@ -410,6 +454,18 @@ var _ = Describe("resourceServer tests", func() { Expect(err).ToNot(HaveOccurred()) Expect(len(res.ContainerResponses)).To(Equal(2)) }) + It("Allocate calls CDI when configured", func() { + rs := resourceServer{resourceName: "fake", socketName: "fake.sock", useCdi: true} + req := &pluginapi.AllocateRequest{ + ContainerRequests: []*pluginapi.ContainerAllocateRequest{nil, nil}, + } + cdi := &cdiMocks.CDI{} + cdi.On("CreateContainerAnnotations", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + rs.cdi = cdi + _, err := rs.Allocate(context.TODO(), req) + Expect(err).ToNot(HaveOccurred()) + cdi.AssertCalled(GinkgoT(), "CreateContainerAnnotations", mock.Anything, mock.Anything, mock.Anything) + }) }) Context("GetInfo", func() { It("GetInfo of plugin", func() { @@ -500,7 +556,7 @@ var _ = Describe("resourceServer tests", func() { }() conf := &types.UserConfig{ResourceName: "fake_test", ResourcePrefix: "rdma", RdmaHcaMax: 100} - obj, err := newResourceServer(conf, fakeDeviceList, true, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, true, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) @@ -564,7 +620,7 @@ var _ = Describe("resourceServer tests", func() { deprecatedSockDir = fs.RootDir conf := &types.UserConfig{ResourceName: "fakename", ResourcePrefix: "rdma", RdmaHcaMax: 100} - obj, err := newResourceServer(conf, fakeDeviceList, false, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, false, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) @@ -593,7 +649,7 @@ var _ = Describe("resourceServer tests", func() { activeSockDir = fs.RootDir conf := &types.UserConfig{ResourceName: "fakename", ResourcePrefix: "rdma", RdmaHcaMax: 100} - obj, err := newResourceServer(conf, fakeDeviceList, true, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, true, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) @@ -623,7 +679,7 @@ var _ = Describe("resourceServer tests", func() { deprecatedSockDir = fs.RootDir conf := &types.UserConfig{ResourceName: "fakename", ResourcePrefix: "rdma", RdmaHcaMax: 100} - obj, err := newResourceServer(conf, fakeDeviceList, false, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, false, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) @@ -652,7 +708,7 @@ var _ = Describe("resourceServer tests", func() { DescribeTable("allocating", func(req *pluginapi.AllocateRequest, expectedRespLength int, shouldFail bool) { conf := &types.UserConfig{ResourceName: "fakename", ResourcePrefix: "rdma", RdmaHcaMax: 100} - obj, err := newResourceServer(conf, fakeDeviceList, true, "socket") + obj, err := newResourceServer(conf, fakeDeviceList, true, "socket", false) Expect(err).ToNot(HaveOccurred()) rs := obj.(*resourceServer) diff --git a/pkg/types/types.go b/pkg/types/types.go index 3155833..3946686 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -1,3 +1,21 @@ +/*---------------------------------------------------- + + 2023 NVIDIA CORPORATION & AFFILIATES + + Licensed under the Apache License, Version 2.0 (the License); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an AS IS BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------------------------*/ + package types import (