don't set default cgroup option if reserved-cpus provided via kubelet…

files/bootstrap.sh

@@ -222,6 +222,13 @@ ENABLE_LOCAL_OUTPOST="${ENABLE_LOCAL_OUTPOST:-}"
 CLUSTER_ID="${CLUSTER_ID:-}"
 LOCAL_DISKS="${LOCAL_DISKS:-}"
 
+##allow --reserved-cpus options via kubelet arg directly. Disable default reserved cgroup option in such cases
+USE_RESERVED_CGROUPS=true
+if [[ ${KUBELET_EXTRA_ARGS} == *'--reserved-cpus'* ]]; then
+  USE_RESERVED_CGROUPS=false
+  log "INFO: --kubelet-extra-args includes --reserved-cpus, so kube/system-reserved cgroups will not be used."
+fi
+
 if [[ ! -z ${LOCAL_DISKS} ]]; then
   setup-local-disks "${LOCAL_DISKS}"
 fi
@@ -565,8 +572,11 @@ if [[ "$CONTAINER_RUNTIME" = "containerd" ]]; then
   sudo sed -i s,SANDBOX_IMAGE,$PAUSE_CONTAINER,g /etc/eks/containerd/containerd-config.toml
 
   echo "$(jq '.cgroupDriver="systemd"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}"
-  echo "$(jq '.systemReservedCgroup="/system"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}"
-  echo "$(jq '.kubeReservedCgroup="/runtime"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}"
+  ##allow --reserved-cpus options via kubelet arg directly. Disable default reserved cgroup option in such cases
+  if [[ "${USE_RESERVED_CGROUPS}" = true ]]; then
+    echo "$(jq '.systemReservedCgroup="/system"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}"
+    echo "$(jq '.kubeReservedCgroup="/runtime"' "${KUBELET_CONFIG}")" > "${KUBELET_CONFIG}"
+  fi
 
   # Check if the containerd config file is the same as the one used in the image build.
   # If different, then restart containerd w/ proper config

test/cases/reserved-cpus-kubelet-arg.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "-> Should not set systemReservedCgroup and kubeReservedCgroup when --reserved-cpus is set with containerd"
+exit_code=0
+export KUBELET_VERSION=v1.24.15-eks-ba74326
+/etc/eks/bootstrap.sh \
+  --b64-cluster-ca dGVzdA== \
+  --apiserver-endpoint http://my-api-endpoint \
+  --kubelet-extra-args '--node-labels=cnf=cnf1 --reserved-cpus=0-3 --cpu-manager-policy=static' \
+  test || exit_code=$?
+
+if [[ ${exit_code} -ne 0 ]]; then
+  echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+  exit 1
+fi
+
+KUBELET_CONFIG=/etc/kubernetes/kubelet/kubelet-config.json
+if grep -q systemReservedCgroup ${KUBELET_CONFIG}; then
+  echo "❌ Test Failed: expected systemReservedCgroup to be absent in ${KUBELET_CONFIG}.Found: $(grep systemReservedCgroup ${KUBELET_CONFIG})"
+  exit 1
+fi
+
+if grep -q kubeReservedCgroup ${KUBELET_CONFIG}; then
+  echo "❌ Test Failed: expected kubeReservedCgroup to be absent ${KUBELET_CONFIG}.Found: $(grep kubeReservedCgroup ${KUBELET_CONFIG})"
+  exit 1
+fi
+
+echo "-> Should set systemReservedCgroup and kubeReservedCgroup when --reserved-cpus is not set with containerd"
+exit_code=0
+export KUBELET_VERSION=v1.24.15-eks-ba74326
+/etc/eks/bootstrap.sh \
+  --b64-cluster-ca dGVzdA== \
+  --apiserver-endpoint http://my-api-endpoint \
+  test || exit_code=$?
+
+if [[ ${exit_code} -ne 0 ]]; then
+  echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+  exit 1
+fi
+
+if ! $(grep -q systemReservedCgroup ${KUBELET_CONFIG}); then
+  echo "❌ Test Failed: expected systemReservedCgroup to be present in ${KUBELET_CONFIG}. Found: $(grep systemReservedCgroup ${KUBELET_CONFIG})"
+  exit 1
+fi
+
+if ! $(grep -q kubeReservedCgroup ${KUBELET_CONFIG}); then
+  echo "❌ Test Failed: expected kubeReservedCgroup to be present ${KUBELET_CONFIG}.Found: $(grep kubeReservedCgroup ${KUBELET_CONFIG})"
+  exit 1
+fi
+
+echo "-> Should set systemReservedCgroup and kubeReservedCgroup when --reserved-cpus is set with dockerd"
+exit_code=0
+export KUBELET_VERSION=v1.23.15-eks-ba74326
+/etc/eks/bootstrap.sh \
+  --b64-cluster-ca dGVzdA== \
+  --apiserver-endpoint http://my-api-endpoint \
+  test || exit_code=$?
+
+if [[ ${exit_code} -ne 0 ]]; then
+  echo "❌ Test Failed: expected a non-zero exit code but got '${exit_code}'"
+  exit 1
+fi
+
+if ! $(grep -q systemReservedCgroup ${KUBELET_CONFIG}); then
+  echo "❌ Test Failed: expected systemReservedCgroup to be present in ${KUBELET_CONFIG}.Found: $(grep systemReservedCgroup ${KUBELET_CONFIG})"
+  exit 1
+fi
+
+if ! $(grep -q kubeReservedCgroup ${KUBELET_CONFIG}); then
+  echo "❌ Test Failed: expected kubeReservedCgroup to be present ${KUBELET_CONFIG}.Found: $(grep kubeReservedCgroup ${KUBELET_CONFIG})"
+  exit 1
+fi