spiderpool-agent: support to configure the sysctl config

Signed-off-by: cyclinder <qifeng.guo@daocloud.io>

charts/spiderpool/README.md

@@ -283,6 +283,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
 | `spiderpoolAgent.resources.requests.cpu`                                             | the cpu requests of spiderpoolAgent pod                                                          | `100m`                                     |
 | `spiderpoolAgent.resources.requests.memory`                                          | the memory requests of spiderpoolAgent pod                                                       | `128Mi`                                    |
 | `spiderpoolAgent.securityContext`                                                    | the security Context of spiderpoolAgent pod                                                      | `{}`                                       |
+| `spiderpoolAgent.sysctlConfigs`                                                      | the sysctl configs of spiderpoolAgent pod                                                        | `{}`                                       |
 | `spiderpoolAgent.httpPort`                                                           | the http Port for spiderpoolAgent, for health checking                                           | `5710`                                     |
 | `spiderpoolAgent.healthChecking.startupProbe.failureThreshold`                       | the failure threshold of startup probe for spiderpoolAgent health checking                       | `60`                                       |
 | `spiderpoolAgent.healthChecking.startupProbe.periodSeconds`                          | the period seconds of startup probe for spiderpoolAgent health checking                          | `2`                                        |

charts/spiderpool/templates/configmap.yaml

@@ -30,6 +30,12 @@ data:
       enabled: {{ .Values.dra.enabled }}
       cdiRootPath: {{ .Values.dra.cdiRootPath }}
       hostDevicePath: {{ .Values.dra.hostDevicePath }}
+    {{- if .Values.spiderpoolAgent.sysctlConfigs }}
+    sysctlConfigs:
+      {{- with .Values.spiderpoolAgent.sysctlConfigs }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+    {{- end }}
 {{- if .Values.multus.multusCNI.install }}
 ---
 kind: ConfigMap

charts/spiderpool/values.yaml

@@ -454,6 +454,9 @@ spiderpoolAgent:
   securityContext: {}
   # runAsUser: 0
 
+  ## @param spiderpoolAgent.sysctlConfigs the sysctl configs of spiderpoolAgent pod
+  sysctlConfigs: {}
+
   ## @param spiderpoolAgent.httpPort the http Port for spiderpoolAgent, for health checking
   httpPort: 5710
 

cmd/spiderpool-agent/cmd/daemon.go

@@ -75,6 +75,8 @@ func DaemonMain() {
 	}
 	logger.Sugar().Infof("Spiderpool-agent config: %+v", agentContext.Cfg)
 
+	// Setup sysctl
+
 	// Set up gops.
 	if agentContext.Cfg.GopsListenPort != "" {
 		address := "127.0.0.1:" + agentContext.Cfg.GopsListenPort

cmd/spiderpool-controller/cmd/daemon.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/google/gops/agent"
 	"github.com/grafana/pyroscope-go"
+	"go.uber.org/zap"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
@@ -35,6 +36,7 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/manager/spidercliamparameter"
 	"github.com/spidernet-io/spiderpool/pkg/multuscniconfig"
 	"github.com/spidernet-io/spiderpool/pkg/namespacemanager"
+	"github.com/spidernet-io/spiderpool/pkg/networking/sysctl"
 	"github.com/spidernet-io/spiderpool/pkg/nodemanager"
 	"github.com/spidernet-io/spiderpool/pkg/openapi"
 	"github.com/spidernet-io/spiderpool/pkg/podmanager"
@@ -81,6 +83,11 @@ func DaemonMain() {
 	}
 	logger.Sugar().Infof("Spiderpool-controller config: %+v", controllerContext.Cfg)
 
+	// setup sysctls
+	if err := sysctlConfig(controllerContext.Cfg.SysctlConfig); err != nil {
+		logger.Sugar().Fatal(err)
+	}
+
 	// Set up gops.
 	if controllerContext.Cfg.GopsListenPort != "" {
 		address := "127.0.0.1:" + controllerContext.Cfg.GopsListenPort
@@ -605,3 +612,25 @@ func checkWebhookReady() {
 		break
 	}
 }
+
+// sysctlConfig set all sysctl config from given user configs.
+func sysctlConfig(configSysctls map[string]string) error {
+	if configSysctls == nil {
+		configSysctls = make(map[string]string)
+	}
+
+	// append default sysctl config
+	for _, sc := range sysctl.DefaultSysctlConfig {
+		if _, ok := configSysctls[sc.Name]; !ok {
+			configSysctls[sc.Name] = sc.Value
+		}
+	}
+
+	for sc, value := range configSysctls {
+		logger.Debug("Setup sysctl", zap.String("sysctl", sc), zap.String("value", value))
+		if err := sysctl.SetSysctl(sc, value); err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/networking/sysctl/sysctl.go

@@ -5,11 +5,38 @@ package sysctl
 
 import (
 	"fmt"
+	"os"
+	"strings"
+
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
-	"os"
 )
 
+// DefaultSysctlConfig is the default sysctl config for the node
+var DefaultSysctlConfig = []struct {
+	Name  string
+	Value string
+}{
+	{
+		Name:  "net.ipv4.neigh.default.gc_thresh3",
+		Value: "8192",
+	},
+	{
+		Name:  "net.ipv6.neigh.default.gc_thresh3",
+		Value: "8192",
+	},
+	{
+		Name:  "net.ipv4.conf.all.arp_notify",
+		Value: "1",
+	}, {
+		Name:  "net.ipv4.conf.all.forwarding",
+		Value: "1",
+	}, {
+		Name:  "net.ipv6.conf.all.forwarding",
+		Value: "1",
+	},
+}
+
 // SysctlRPFilter set rp_filter value for host netns and specify netns
 func SysctlRPFilter(netns ns.NetNS, value int32) error {
 	var err error
@@ -77,3 +104,21 @@ func EnableIpv6Sysctl(netns ns.NetNS) error {
 	})
 	return err
 }
+
+func SetSysctl(sysConfig string, value string) error {
+	// sysConfig: net.ipv6.neigh.default.gc_thresh3
+	// to: net/ipv6/neigh/default/gc_thresh3
+	sysConfig = strings.ReplaceAll(sysConfig, ".", "/")
+	sysConfig = fmt.Sprintf("/proc/sys/%s", sysConfig)
+
+	_, err := os.Stat(sysConfig)
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("invalid sysctl %s, failed to set: %w", sysConfig, err)
+	}
+
+	if _, err := sysctl.Sysctl(sysConfig, value); err != nil {
+		return fmt.Errorf("error to set sysctl %s to %s: %w", sysConfig, value, err)
+	}
+
+	return nil
+}

pkg/types/k8s.go

@@ -109,7 +109,6 @@ type AutoPoolProperty struct {
 }
 
 type SpiderpoolConfigmapConfig struct {
-	DraConfig                                     `yaml:"dra"`
 	IpamUnixSocketPath                            string `yaml:"ipamUnixSocketPath"`
 	EnableIPv4                                    bool   `yaml:"enableIPv4"`
 	EnableIPv6                                    bool   `yaml:"enableIPv6"`
@@ -118,6 +117,8 @@ type SpiderpoolConfigmapConfig struct {
 	EnableSpiderSubnet                            bool   `yaml:"enableSpiderSubnet"`
 	EnableAutoPoolForApplication                  bool   `yaml:"enableAutoPoolForApplication"`
 	ClusterSubnetAutoPoolDefaultRedundantIPNumber int    `yaml:"clusterSubnetAutoPoolDefaultRedundantIPNumber"`
+	DraConfig                                     `yaml:"dra"`
+	SysctlConfig                                  map[string]string `yaml:"sysctlConfig"`
 }
 
 type DraConfig struct {