Skip to content

Commit

Permalink
add k8s-pure cni
Browse files Browse the repository at this point in the history
Signed-off-by: forrestchen <forrestchen@tencent.com>
  • Loading branch information
ChenLingPeng committed Apr 24, 2020
1 parent cbc450c commit 42e6273
Show file tree
Hide file tree
Showing 12 changed files with 219 additions and 47 deletions.
2 changes: 1 addition & 1 deletion build/docker/galaxy/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@ LABEL description="This Dockerfile is written for galaxy"
WORKDIR /root/
RUN yum install -y iproute iptables ipset
COPY host-local loopback /opt/cni/galaxy/bin/
COPY galaxy-k8s-sriov galaxy-k8s-vlan galaxy-bridge galaxy-flannel galaxy-veth galaxy-sdn tke-route-eni /opt/cni/galaxy/bin/
COPY galaxy-k8s-sriov galaxy-k8s-vlan galaxy-k8s-pure galaxy-bridge galaxy-flannel galaxy-veth galaxy-sdn tke-route-eni /opt/cni/galaxy/bin/
COPY galaxy /usr/bin/
2 changes: 1 addition & 1 deletion build/lib/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ function build::galaxy() {
echo "Building plugins"

# build galaxy cni plugins
PLUGINS="${PKG}/cni/k8s-vlan ${PKG}/cni/sdn ${PKG}/cni/veth ${PKG}/cni/k8s-sriov"
PLUGINS="${PKG}/cni/k8s-vlan ${PKG}/cni/sdn ${PKG}/cni/veth ${PKG}/cni/k8s-sriov ${PKG}/cni/k8s-pure"
for d in ${PLUGINS}; do
plugin=$(basename $d)
echo " " ${plugin}
Expand Down
102 changes: 102 additions & 0 deletions cni/k8s-pure/k8s_pure.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package main

import (
"encoding/json"
"fmt"
"net"
"runtime"

"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
t020 "github.com/containernetworking/cni/pkg/types/020"
"github.com/containernetworking/cni/pkg/version"
"github.com/vishvananda/netlink"
"tkestack.io/galaxy/cni/ipam"
"tkestack.io/galaxy/pkg/network"
"tkestack.io/galaxy/pkg/network/vlan"
"tkestack.io/galaxy/pkg/utils"
)

func init() {
// this ensures that main runs only on main thread (thread group leader).
// since namespace ops (unshare, setns) are done for a single thread, we
// must ensure that the goroutine does not jump from OS thread to thread
runtime.LockOSThread()
}

func main() {
skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
}

func cmdDel(args *skel.CmdArgs) error {
if err := utils.DeleteAllVeth(args.Netns); err != nil {
return err
}
return nil
}

func cmdAdd(args *skel.CmdArgs) error {
conf := vlan.NetConf{}
if err := json.Unmarshal(args.StdinData, &conf); err != nil {
return fmt.Errorf("conf error: %v", err)
}
vlanIds, results, err := ipam.Allocate(conf.IPAM.Type, args)
if err != nil {
return fmt.Errorf("allocate failed: %v", err)
}
if err := utils.UnSetArpIgnore("all"); err != nil {
return err
}
if err := utils.EnableNonlocalBind(); err != nil {
return err
}
ifName := args.IfName
ifIndex := 0
for i := range vlanIds {
vlanId := vlanIds[i]
result, err := t020.GetResult(results[i])
if err != nil {
return fmt.Errorf("result convert failed: %v", err)
}
device := conf.Device
// fixme: make route configurable
if i != 0 {
result.IP4.Routes = []types.Route{{
Dst: net.IPNet{
IP: result.IP4.IP.IP.Mask(result.IP4.IP.Mask),
Mask: result.IP4.IP.Mask,
},
}}
}
var masterDevice netlink.Link
if masterDevice, err = vlan.SetupVlanInPureMode(device, vlanId); err != nil {
return fmt.Errorf("failed setup vlan: %v", err)
}
suffix := ""
if i != 0 {
suffix = fmt.Sprintf("-%d", i+1)
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
if args.IfName == ifName {
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
}
}
v4Addr, err := netlink.AddrList(masterDevice, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("error getting ipv4 address %v", err)
}
filteredAddr := network.FilterLoopbackAddr(v4Addr)
var src net.IP
if len(filteredAddr) > 0 {
src = filteredAddr[0].IP
}
if err := utils.VethConnectsHostWithContainer(result, args, "", suffix, src); err != nil {
return fmt.Errorf("veth connect failed: %v", err)
}
utils.SendGratuitousARP(masterDevice.Attrs().Name, result.IP4.IP.IP.String(), "", conf.GratuitousArpRequest)
}
args.IfName = ifName
result, _ := t020.GetResult(results[0])
return result.Print()
}
2 changes: 1 addition & 1 deletion cni/k8s-vlan/k8s_vlan.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ func setupVlanDevice(result020s []*t020.Result, vlanIds []uint16, args *skel.Cmd
args.IfName = fmt.Sprintf("eth%d", ifIndex)
}
}
if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix); err != nil {
if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix, nil); err != nil {
return err
}
_ = utils.SendGratuitousARP(args.IfName, result020s[0].IP4.IP.IP.String(), args.Netns, d.GratuitousArpRequest)
Expand Down
25 changes: 1 addition & 24 deletions cni/veth/veth.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,30 +60,7 @@ func addHostRoute(containerIP *net.IPNet, vethHostName string, src string) error
return err
}
s := net.ParseIP(src)
if err = netlink.RouteAdd(&netlink.Route{
LinkIndex: vethHost.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: containerIP,
Gw: nil,
Src: s,
}); err != nil {
if s != nil {
// compatible change for old kernel which does not support src option
if err1 := netlink.RouteAdd(&netlink.Route{
LinkIndex: vethHost.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: containerIP,
Gw: nil,
}); err1 != nil {
return fmt.Errorf("failed to add route '%v dev %v for old linux kernel': %v. With src option err: %v",
containerIP, vethHostName, err1, err)
} else {
return nil
}
}
return fmt.Errorf("failed to add route '%v dev %v src %v': %v", containerIP, vethHostName, s.String(), err)
}
return nil
return utils.AddHostRoute(containerIP, vethHost, s)
}

// #lizard forgives
Expand Down
3 changes: 2 additions & 1 deletion doc/galaxy-config.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ Galaxy support multiple default networks and configures them to pods according t
{"name":"galaxy-flannel", "type":"galaxy-flannel", "delegate":{"type":"galaxy-veth"}, "subnetFile":"/run/flannel
/subnet.env"},
{"name":"galaxy-k8s-vlan", "type":"galaxy-k8s-vlan", "device":"eth1", "default_bridge_name": "br0"},
{"name": "galaxy-k8s-sriov", "type": "galaxy-k8s-sriov", "device": "eth1", "vf_num": 10}
{"name": "galaxy-k8s-sriov", "type": "galaxy-k8s-sriov", "device": "eth1", "vf_num": 10},
{"name":"galaxy-k8s-pure", "type":"galaxy-k8s-pure", "device":"eth1"}
],
"DefaultNetworks": ["galaxy-flannel"],
"ENIIPNetwork": "galaxy-k8s-vlan"
Expand Down
2 changes: 1 addition & 1 deletion hack/build-native.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ cd $ROOT
PKG=tkestack.io/galaxy
BIN_PREFIX="galaxy"
# build galaxy cni plugins
PLUGINS="$GOPATH/src/${PKG}/cni/k8s-vlan $GOPATH/src/${PKG}/cni/sdn $GOPATH/src/${PKG}/cni/veth $GOPATH/src/${PKG}/cni/k8s-sriov"
PLUGINS="$GOPATH/src/${PKG}/cni/k8s-vlan $GOPATH/src/${PKG}/cni/sdn $GOPATH/src/${PKG}/cni/veth $GOPATH/src/${PKG}/cni/k8s-sriov $GOPATH/src/${PKG}/cni/k8s-pure"
for d in $PLUGINS; do
if [ -d $d ]; then
plugin=$(basename $d)
Expand Down
3 changes: 3 additions & 0 deletions pkg/api/cniutil/cni.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ func CmdAdd(cmdArgs *skel.CmdArgs, networkInfos []*NetworkInfo) (types.Result, e
for idx, networkInfo := range networkInfos {
//append additional args from network info
cmdArgs.Args = strings.TrimRight(fmt.Sprintf("%s;%s", cmdArgs.Args, BuildCNIArgs(networkInfo.Args)), ";")
if result != nil {
networkInfo.Conf["prevResult"] = result
}
result, err = DelegateAdd(networkInfo.Conf, cmdArgs, networkInfo.IfName)
if err != nil {
//fail to add cni, then delete all established CNIs recursively
Expand Down
17 changes: 4 additions & 13 deletions pkg/galaxy/galaxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
glog "k8s.io/klog"
"tkestack.io/galaxy/pkg/api/docker"
Expand Down Expand Up @@ -150,19 +149,11 @@ func (g *Galaxy) Stop() error {
}

func (g *Galaxy) initk8sClient() {
var clientConfig *rest.Config
var err error
if g.Master != "" || g.KubeConf != "" {
clientConfig, err = clientcmd.BuildConfigFromFlags(g.Master, g.KubeConf)
if err != nil {
glog.Fatalf("Invalid client config: %v", err)
}
} else {
clientConfig, err = rest.InClusterConfig()
if err != nil {
glog.Fatalf("Init InClient config failed: %v", err)
}
clientConfig, err := clientcmd.BuildConfigFromFlags(g.Master, g.KubeConf)
if err != nil {
glog.Fatalf("Invalid client config: %v", err)
}

clientConfig.QPS = 1000.0
clientConfig.Burst = 2000
glog.Infof("QPS: %e, Burst: %d", clientConfig.QPS, clientConfig.Burst)
Expand Down
63 changes: 63 additions & 0 deletions pkg/network/vlan/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package vlan

import (
"fmt"

"github.com/vishvananda/netlink"
"tkestack.io/galaxy/pkg/utils"
)

// SetupVlan is used in pure veth mode, in this mode, vlan will not attach to bridge device
func SetupVlanInPureMode(ethDevice string, vlanId uint16) (netlink.Link, error) {
if vlanId == 0 {
device, err := netlink.LinkByName(ethDevice)
if err != nil {
return nil, fmt.Errorf("device %v not found", err)
}
if err := utils.UnSetArpIgnore(ethDevice); err != nil {
return nil, err
}
if err := utils.SetProxyArp(ethDevice); err != nil {
return nil, err
}
return device, nil
}
vlanDevice, err := findVlanDevice(vlanId)
if err != nil {
return nil, fmt.Errorf("find vlan device failed: %v", err)
}
if err = netlink.LinkSetUp(vlanDevice); err != nil {
return nil, fmt.Errorf("failed set vlan %v up: %v", vlanDevice.Attrs().Name, err)
}
if err := utils.UnSetArpIgnore(vlanDevice.Attrs().Name); err != nil {
return nil, err
}
return vlanDevice, utils.SetProxyArp(vlanDevice.Attrs().Name)
}

func findVlanDevice(vlanID uint16) (netlink.Link, error) {
links, err := netlink.LinkList()
if err != nil {
return nil, err
}
var result netlink.Link
for _, link := range links {
if link.Type() == "vlan" {
if vlan, ok := link.(*netlink.Vlan); !ok {
return nil, fmt.Errorf("vlan device type case error: %T", link)
} else {
if vlan.VlanId == int(vlanID) {
if result == nil {
result = link
} else {
return nil, fmt.Errorf("found 2 device with same vlanID %d: %s, %s", vlanID, result.Attrs().Name, vlan.Name)
}
}
}
}
}
if result == nil {
return nil, fmt.Errorf("no vlan device with vlanID %d", vlanID)
}
return result, nil
}
43 changes: 39 additions & 4 deletions pkg/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import (

"github.com/containernetworking/cni/pkg/skel"
t020 "github.com/containernetworking/cni/pkg/types/020"
"github.com/containernetworking/plugins/pkg/ip"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/vishvananda/netlink"
"tkestack.io/galaxy/pkg/api/cniutil"
Expand Down Expand Up @@ -255,15 +254,40 @@ func CreateVeth(containerID string, mtu int, suffix string) (netlink.Link, netli
return host, sbox, nil
}

func AddHostRoute(containerIP *net.IPNet, host netlink.Link, src net.IP) error {
if err := netlink.RouteAdd(&netlink.Route{
LinkIndex: host.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: containerIP,
Gw: nil,
Src: src,
}); err != nil {
if src != nil {
// compatible change for old kernel which does not support src option such as tlinux 0041 0042
if err1 := netlink.RouteAdd(&netlink.Route{
LinkIndex: host.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: containerIP,
Gw: nil,
}); err1 != nil {
return fmt.Errorf("failed to add route '%v dev %v for old linux kernel': %v. With src option err: %v", containerIP, host.Attrs().Name, err1, err)
} else {
return nil
}
}
return fmt.Errorf("failed to add route '%v dev %v src %v': %v", containerIP, host.Attrs().Name, src.String(), err)
}
return nil
}

// #lizard forgives
// VethConnectsHostWithContainer creates veth device pairs and connects container with host
// If bridgeName specified, it attaches host side veth device to the bridge
func VethConnectsHostWithContainer(result *t020.Result, args *skel.CmdArgs, bridgeName string, suffix string) error {
func VethConnectsHostWithContainer(result *t020.Result, args *skel.CmdArgs, bridgeName string, suffix string, src net.IP) error {
host, sbox, err := CreateVeth(args.ContainerID, 1500, suffix)
if err != nil {
return err
}
// nolint: errcheck
defer func() {
if err != nil {
if host != nil {
Expand Down Expand Up @@ -292,7 +316,7 @@ func VethConnectsHostWithContainer(result *t020.Result, args *skel.CmdArgs, brid
if bridgeName == "" {
desIP := result.IP4.IP.IP
ipn := net.IPNet{IP: desIP, Mask: net.CIDRMask(32, 32)}
if err = ip.AddRoute(&ipn, nil, host); err != nil {
if err = AddHostRoute(&ipn, host, src); err != nil {
return err
}
}
Expand Down Expand Up @@ -403,6 +427,12 @@ func configSboxDevice(result *t020.Result, args *skel.CmdArgs, sbox netlink.Link
if err := netlink.LinkSetName(sbox, args.IfName); err != nil {
return fmt.Errorf("failed to rename sbox device %q to %q: %v", sbox.Attrs().Name, args.IfName, err)
}
// disable rp_filter is needed when there're multi network device in container and host,
// the arp request from host will pick source ip un-determined,
// so device in container should disable rp_filter to answer this request
if err := DisableRpFilter(args.IfName); err != nil {
return fmt.Errorf("failed disable rp_filter to dev %s: %v", args.IfName, err)
}
// Add IP and routes to sbox, including default route
return cniutil.ConfigureIface(args.IfName, result)
})
Expand All @@ -413,6 +443,11 @@ func SetProxyArp(dev string) error {
return ioutil.WriteFile(file, []byte("1\n"), 0644)
}

func DisableRpFilter(dev string) error {
file := fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/rp_filter", dev)
return ioutil.WriteFile(file, []byte("0\n"), 0644)
}

func UnSetArpIgnore(dev string) error {
file := fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/arp_ignore", dev)
return ioutil.WriteFile(file, []byte("0\n"), 0644)
Expand Down
2 changes: 1 addition & 1 deletion tools/network/setupvlan.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func main() {
},
}},
},
}, &skel.CmdArgs{Netns: *flagNetns, IfName: "eth0"}, bridgeName, ""); err != nil {
}, &skel.CmdArgs{Netns: *flagNetns, IfName: "eth0"}, bridgeName, "", nil); err != nil {
glog.Fatalf("Error creating veth %v", err)
}
}
Expand Down

0 comments on commit 42e6273

Please sign in to comment.