spiderpool-agent: support to configure the sysctl config

Signed-off-by: cyclinder <qifeng.guo@daocloud.io>

charts/spiderpool/README.md

@@ -283,6 +283,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
 | `spiderpoolAgent.resources.requests.cpu`                                             | the cpu requests of spiderpoolAgent pod                                                          | `100m`                                     |
 | `spiderpoolAgent.resources.requests.memory`                                          | the memory requests of spiderpoolAgent pod                                                       | `128Mi`                                    |
 | `spiderpoolAgent.securityContext`                                                    | the security Context of spiderpoolAgent pod                                                      | `{}`                                       |
+| `spiderpoolAgent.sysctlConfigs`                                                      | the sysctl configs of spiderpoolAgent pod                                                        | `{}`                                       |
 | `spiderpoolAgent.httpPort`                                                           | the http Port for spiderpoolAgent, for health checking                                           | `5710`                                     |
 | `spiderpoolAgent.healthChecking.startupProbe.failureThreshold`                       | the failure threshold of startup probe for spiderpoolAgent health checking                       | `60`                                       |
 | `spiderpoolAgent.healthChecking.startupProbe.periodSeconds`                          | the period seconds of startup probe for spiderpoolAgent health checking                          | `2`                                        |

charts/spiderpool/templates/configmap.yaml

@@ -30,6 +30,12 @@ data:
       enabled: {{ .Values.dra.enabled }}
       cdiRootPath: {{ .Values.dra.cdiRootPath }}
       hostDevicePath: {{ .Values.dra.hostDevicePath }}
+    {{- if .Values.spiderpoolAgent.sysctlConfigs }}
+    sysctlConfigs:
+      {{- with .Values.spiderpoolAgent.sysctlConfigs }}
+      {{- toYaml . | nindent 6 }}
+      {{- end }}
+    {{- end }}
 {{- if .Values.multus.multusCNI.install }}
 ---
 kind: ConfigMap

charts/spiderpool/values.yaml

@@ -454,6 +454,9 @@ spiderpoolAgent:
   securityContext: {}
   # runAsUser: 0
 
+  ## @param spiderpoolAgent.sysctlConfigs the sysctl configs of spiderpoolAgent pod
+  sysctlConfigs: {}
+
   ## @param spiderpoolAgent.httpPort the http Port for spiderpoolAgent, for health checking
   httpPort: 5710
 

cmd/spiderpool-agent/cmd/daemon.go

@@ -29,6 +29,7 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/kubevirtmanager"
 	"github.com/spidernet-io/spiderpool/pkg/logutils"
 	"github.com/spidernet-io/spiderpool/pkg/namespacemanager"
+	"github.com/spidernet-io/spiderpool/pkg/networking/sysctl"
 	"github.com/spidernet-io/spiderpool/pkg/nodemanager"
 	"github.com/spidernet-io/spiderpool/pkg/openapi"
 	"github.com/spidernet-io/spiderpool/pkg/podmanager"
@@ -75,6 +76,11 @@ func DaemonMain() {
 	}
 	logger.Sugar().Infof("Spiderpool-agent config: %+v", agentContext.Cfg)
 
+	// setup sysctls
+	if err := sysctlConfig(agentContext.Cfg.EnableIPv4, agentContext.Cfg.EnableIPv6, agentContext.Cfg.SysctlConfig); err != nil {
+		logger.Sugar().Fatal(err)
+	}
+
 	// Set up gops.
 	if agentContext.Cfg.GopsListenPort != "" {
 		address := "127.0.0.1:" + agentContext.Cfg.GopsListenPort
@@ -430,3 +436,43 @@ func initAgentServiceManagers(ctx context.Context) {
 		logger.Info("Feature SpiderSubnet is disabled")
 	}
 }
+
+// sysctlConfig set all sysctl configs, including default sysctl configs and
+// user defined sysctl configs. Notice: ignore not exist sysctl configs as
+// possible.
+func sysctlConfig(enableIPv4, enableIPv6 bool, configSysctls map[string]string) error {
+	// setup default sysctl config
+	for _, sc := range sysctl.DefaultSysctlConfig {
+		if (enableIPv4 && sc.IsIPv4) || (enableIPv6 && sc.IsIPv6) {
+			logger.Debug("Setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value))
+			err := sysctl.SetSysctl(sc.Name, sc.Value)
+			if err == nil {
+				logger.Debug("success to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value))
+				continue
+			}
+
+			if !errors.Is(err, os.ErrNotExist) {
+				logger.Error("failed to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err))
+				return err
+			}
+			logger.Warn("skip to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err))
+		}
+	}
+
+	// setup user sysctl config
+	for sc, value := range configSysctls {
+		logger.Debug("Setup sysctl", zap.String("sysctl", sc), zap.String("value", value))
+		err := sysctl.SetSysctl(sc, value)
+		if err == nil {
+			logger.Debug("success to setup sysctl", zap.String("sysctl", sc), zap.String("value", value))
+			continue
+		}
+
+		if !errors.Is(err, os.ErrNotExist) {
+			logger.Error("failed to setup sysctl", zap.String("sysctl", sc), zap.String("value", value), zap.Error(err))
+			return err
+		}
+		logger.Sugar().Warnf("skip to setup sysctl %s: %v", sc, err)
+	}
+	return nil
+}

pkg/networking/sysctl/sysctl.go

@@ -5,11 +5,53 @@ package sysctl
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
-	"os"
 )
 
+// DefaultSysctlConfig is the default sysctl config for the node
+var DefaultSysctlConfig = []struct {
+	Name           string
+	Value          string
+	IsIPv4, IsIPv6 bool
+}{
+	// In order to avoid large-scale cluster arp_table overflow, resulting in
+	// pods not being able to communicate or pods not being able to start due
+	// to the inability to insert static arp table entries, it is necessary
+	// to appropriately increase and adjust its value. more details see:
+	// https://github.com/spidernet-io/spiderpool/issues/3587
+	{
+		Name:   "net.ipv4.neigh.default.gc_thresh3",
+		Value:  "8192",
+		IsIPv4: true,
+	},
+	{
+		// this sysctl may not be available at low kernel levels,
+		// so we'll ignore it at this point.
+		Name:   "net.ipv6.neigh.default.gc_thresh3",
+		Value:  "8192",
+		IsIPv6: true,
+	},
+	// send gratitous ARP when device or address change
+	{
+		Name:   "net.ipv4.conf.all.arp_notify",
+		Value:  "1",
+		IsIPv4: true,
+	}, {
+		Name:   "net.ipv4.conf.all.forwarding",
+		Value:  "1",
+		IsIPv4: true,
+	}, {
+		Name:   "net.ipv6.conf.all.forwarding",
+		Value:  "1",
+		IsIPv6: true,
+	},
+}
+
 // SysctlRPFilter set rp_filter value for host netns and specify netns
 func SysctlRPFilter(netns ns.NetNS, value int32) error {
 	var err error
@@ -77,3 +119,20 @@ func EnableIpv6Sysctl(netns ns.NetNS) error {
 	})
 	return err
 }
+
+func SetSysctl(sysConfig string, value string) error {
+	// sysConfig: net.ipv6.neigh.default.gc_thresh3
+	// to: net/ipv6/neigh/default/gc_thresh3
+	sysConfig = strings.ReplaceAll(sysConfig, ".", "/")
+
+	_, err := os.Stat(filepath.Join("/proc/sys", sysConfig))
+	if err != nil {
+		return err
+	}
+
+	if _, err := sysctl.Sysctl(sysConfig, value); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/types/k8s.go

@@ -109,7 +109,6 @@ type AutoPoolProperty struct {
 }
 
 type SpiderpoolConfigmapConfig struct {
-	DraConfig                                     `yaml:"dra"`
 	IpamUnixSocketPath                            string `yaml:"ipamUnixSocketPath"`
 	EnableIPv4                                    bool   `yaml:"enableIPv4"`
 	EnableIPv6                                    bool   `yaml:"enableIPv6"`
@@ -118,6 +117,8 @@ type SpiderpoolConfigmapConfig struct {
 	EnableSpiderSubnet                            bool   `yaml:"enableSpiderSubnet"`
 	EnableAutoPoolForApplication                  bool   `yaml:"enableAutoPoolForApplication"`
 	ClusterSubnetAutoPoolDefaultRedundantIPNumber int    `yaml:"clusterSubnetAutoPoolDefaultRedundantIPNumber"`
+	DraConfig                                     `yaml:"dra"`
+	SysctlConfig                                  map[string]string `yaml:"sysctlConfig"`
 }
 
 type DraConfig struct {