-
Notifications
You must be signed in to change notification settings - Fork 76
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
spiderpool-agent: support to configure the sysctl config for node #3772
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -29,6 +29,7 @@ import ( | |
"github.com/spidernet-io/spiderpool/pkg/kubevirtmanager" | ||
"github.com/spidernet-io/spiderpool/pkg/logutils" | ||
"github.com/spidernet-io/spiderpool/pkg/namespacemanager" | ||
"github.com/spidernet-io/spiderpool/pkg/networking/sysctl" | ||
"github.com/spidernet-io/spiderpool/pkg/nodemanager" | ||
"github.com/spidernet-io/spiderpool/pkg/openapi" | ||
"github.com/spidernet-io/spiderpool/pkg/podmanager" | ||
|
@@ -75,6 +76,15 @@ func DaemonMain() { | |
} | ||
logger.Sugar().Infof("Spiderpool-agent config: %+v", agentContext.Cfg) | ||
|
||
// setup sysctls | ||
if agentContext.Cfg.TuneSysctlConfig { | ||
if err := sysctlConfig(agentContext.Cfg.EnableIPv4, agentContext.Cfg.EnableIPv6); err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 作为 info 级别的日志,打开时,看不到一行 " set syctl " 日志 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这个日志是有的 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
logger.Sugar().Fatal(err) | ||
} | ||
} else { | ||
logger.Sugar().Infof("setSysctlConfig is disabled.") | ||
} | ||
|
||
// Set up gops. | ||
if agentContext.Cfg.GopsListenPort != "" { | ||
address := "127.0.0.1:" + agentContext.Cfg.GopsListenPort | ||
|
@@ -430,3 +440,26 @@ func initAgentServiceManagers(ctx context.Context) { | |
logger.Info("Feature SpiderSubnet is disabled") | ||
} | ||
} | ||
|
||
// sysctlConfig set default sysctl configs,Notice: ignore not exist sysctl configs as | ||
// possible. | ||
func sysctlConfig(enableIPv4, enableIPv6 bool) error { | ||
// setup default sysctl config | ||
for _, sc := range sysctl.DefaultSysctlConfig { | ||
if (enableIPv4 && sc.IsIPv4) || (enableIPv6 && sc.IsIPv6) { | ||
logger.Info("Setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value)) | ||
err := sysctl.SetSysctl(sc.Name, sc.Value) | ||
if err == nil { | ||
logger.Debug("success to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value)) | ||
continue | ||
} | ||
|
||
if !errors.Is(err, os.ErrNotExist) { | ||
logger.Error("failed to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err)) | ||
return err | ||
} | ||
logger.Warn("skip to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err)) | ||
} | ||
} | ||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,11 +5,55 @@ package sysctl | |
|
||
import ( | ||
"fmt" | ||
"os" | ||
"path/filepath" | ||
"strings" | ||
|
||
"github.com/containernetworking/plugins/pkg/ns" | ||
"github.com/containernetworking/plugins/pkg/utils/sysctl" | ||
"os" | ||
) | ||
|
||
// DefaultSysctlConfig is the default sysctl config for the node | ||
var DefaultSysctlConfig = []struct { | ||
Name string | ||
Value string | ||
IsIPv4, IsIPv6 bool | ||
}{ | ||
// In order to avoid large-scale cluster arp_table overflow, resulting in | ||
// pods not being able to communicate or pods not being able to start due | ||
// to the inability to insert static arp table entries, it is necessary | ||
// to appropriately increase and adjust its value. more details see: | ||
// https://github.com/spidernet-io/spiderpool/issues/3587 | ||
{ | ||
cyclinder marked this conversation as resolved.
Show resolved
Hide resolved
|
||
Name: "net.ipv4.neigh.default.gc_thresh3", | ||
// Assuming a node is full of underlay pods (110) and their subnet | ||
// mask is 16 bits ( 2 ^ 8 = 256 IPs), the value is 110 * 256 = 28160 | ||
Value: "28160", | ||
IsIPv4: true, | ||
}, | ||
{ | ||
// this sysctl may not be available at low kernel levels, | ||
// so we'll ignore it at this point. | ||
Name: "net.ipv6.neigh.default.gc_thresh3", | ||
Value: "28160", | ||
IsIPv6: true, | ||
}, | ||
// send gratitous ARP when device or address change | ||
{ | ||
Name: "net.ipv4.conf.all.arp_notify", | ||
Value: "1", | ||
weizhoublue marked this conversation as resolved.
Show resolved
Hide resolved
|
||
IsIPv4: true, | ||
}, { | ||
Name: "net.ipv4.conf.all.forwarding", | ||
Value: "1", | ||
IsIPv4: true, | ||
}, { | ||
Name: "net.ipv6.conf.all.forwarding", | ||
Value: "1", | ||
IsIPv6: true, | ||
}, | ||
} | ||
|
||
// SysctlRPFilter set rp_filter value for host netns and specify netns | ||
func SysctlRPFilter(netns ns.NetNS, value int32) error { | ||
var err error | ||
|
@@ -77,3 +121,20 @@ func EnableIpv6Sysctl(netns ns.NetNS) error { | |
}) | ||
return err | ||
} | ||
|
||
func SetSysctl(sysConfig string, value string) error { | ||
// sysConfig: net.ipv6.neigh.default.gc_thresh3 | ||
// to: net/ipv6/neigh/default/gc_thresh3 | ||
sysConfig = strings.ReplaceAll(sysConfig, ".", "/") | ||
|
||
weizhoublue marked this conversation as resolved.
Show resolved
Hide resolved
|
||
_, err := os.Stat(filepath.Join("/proc/sys", sysConfig)) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if _, err := sysctl.Sysctl(sysConfig, value); err != nil { | ||
return err | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 需要一行 info 级别日志: sysctl 什么 key 设置了 什么值 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
return nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
这个默认是 强制的? 难免有一些 安全风险 和 背锅
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
哦,这个应该要恢复