Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add k8s-pure cni #67

Merged
merged 4 commits into from
Apr 26, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build/docker/galaxy/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@ LABEL description="This Dockerfile is written for galaxy"
WORKDIR /root/
RUN yum install -y iproute iptables ipset
COPY host-local loopback /opt/cni/galaxy/bin/
COPY galaxy-k8s-sriov galaxy-k8s-vlan galaxy-bridge galaxy-flannel galaxy-veth galaxy-sdn tke-route-eni /opt/cni/galaxy/bin/
COPY galaxy-k8s-sriov galaxy-k8s-vlan galaxy-underlay-veth galaxy-bridge galaxy-flannel galaxy-veth galaxy-sdn tke-route-eni /opt/cni/galaxy/bin/
COPY galaxy /usr/bin/
7 changes: 4 additions & 3 deletions build/lib/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,12 @@ function build::galaxy() {
echo "Building plugins"

# build galaxy cni plugins
PLUGINS="${PKG}/cni/k8s-vlan ${PKG}/cni/sdn ${PKG}/cni/veth ${PKG}/cni/k8s-sriov"
PLUGINS="${PKG}/cni/k8s-vlan ${PKG}/cni/sdn ${PKG}/cni/veth ${PKG}/cni/k8s-sriov ${PKG}/cni/underlay/veth"
for d in ${PLUGINS}; do
plugin=$(basename $d)
plugin_dir=${d#"${PKG}/cni/"}
plugin=${plugin_dir//"/"/"-"}
echo " " ${plugin}
go build -o ${BIN_DIR}/${BIN_PREFIX}-${plugin} ${GOBUILD_FLAGS} ${PKG}/cni/${plugin}
go build -o ${BIN_DIR}/${BIN_PREFIX}-${plugin} ${GOBUILD_FLAGS} ${d}
done

# build cni plugins (no galaxy prefix)
Expand Down
2 changes: 1 addition & 1 deletion cni/k8s-vlan/k8s_vlan.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ func setupVlanDevice(result020s []*t020.Result, vlanIds []uint16, args *skel.Cmd
args.IfName = fmt.Sprintf("eth%d", ifIndex)
}
}
if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix); err != nil {
if err := utils.VethConnectsHostWithContainer(result020, args, bridgeName, suffix, nil); err != nil {
return err
}
_ = utils.SendGratuitousARP(args.IfName, result020s[0].IP4.IP.IP.String(), args.Netns, d.GratuitousArpRequest)
Expand Down
102 changes: 102 additions & 0 deletions cni/underlay/veth/veth.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package main

import (
"encoding/json"
"fmt"
"net"
"runtime"

"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
t020 "github.com/containernetworking/cni/pkg/types/020"
"github.com/containernetworking/cni/pkg/version"
"github.com/vishvananda/netlink"
"tkestack.io/galaxy/cni/ipam"
"tkestack.io/galaxy/pkg/network"
"tkestack.io/galaxy/pkg/network/vlan"
"tkestack.io/galaxy/pkg/utils"
)

func init() {
// this ensures that main runs only on main thread (thread group leader).
// since namespace ops (unshare, setns) are done for a single thread, we
// must ensure that the goroutine does not jump from OS thread to thread
runtime.LockOSThread()
}

func main() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

改下文档supported-cnis.md,简单介绍下怎么使用这个cni插件吗?比如需要管理员自己把vlan网卡和路由配置好

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

好的

skel.PluginMain(cmdAdd, cmdDel, version.Legacy)
}

func cmdDel(args *skel.CmdArgs) error {
if err := utils.DeleteAllVeth(args.Netns); err != nil {
return err
}
return nil
}

func cmdAdd(args *skel.CmdArgs) error {
conf := vlan.NetConf{}
if err := json.Unmarshal(args.StdinData, &conf); err != nil {
return fmt.Errorf("conf error: %v", err)
}
vlanIds, results, err := ipam.Allocate(conf.IPAM.Type, args)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

还是调用一次这个veth插件配置多个网卡?可以以后再改吧

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

是的,不然这里涉及到的改动太多,方案也还需讨论

if err != nil {
return fmt.Errorf("allocate failed: %v", err)
}
if err := utils.UnSetArpIgnore("all"); err != nil {
return err
}
if err := utils.EnableNonlocalBind(); err != nil {
return err
}
ifName := args.IfName
ifIndex := 0
for i := range vlanIds {
vlanId := vlanIds[i]
result, err := t020.GetResult(results[i])
if err != nil {
return fmt.Errorf("result convert failed: %v", err)
}
device := conf.Device
// fixme: make route configurable
if i != 0 {
result.IP4.Routes = []types.Route{{
Dst: net.IPNet{
IP: result.IP4.IP.IP.Mask(result.IP4.IP.Mask),
Mask: result.IP4.IP.Mask,
},
}}
}
var masterDevice netlink.Link
if masterDevice, err = vlan.SetupVlanInPureMode(device, vlanId); err != nil {
return fmt.Errorf("failed setup vlan: %v", err)
}
suffix := ""
if i != 0 {
suffix = fmt.Sprintf("-%d", i+1)
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
if args.IfName == ifName {
ifIndex++
args.IfName = fmt.Sprintf("eth%d", ifIndex)
}
}
v4Addr, err := netlink.AddrList(masterDevice, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("error getting ipv4 address %v", err)
}
filteredAddr := network.FilterLoopbackAddr(v4Addr)
var src net.IP
if len(filteredAddr) > 0 {
src = filteredAddr[0].IP
}
if err := utils.VethConnectsHostWithContainer(result, args, "", suffix, src); err != nil {
return fmt.Errorf("veth connect failed: %v", err)
}
utils.SendGratuitousARP(masterDevice.Attrs().Name, result.IP4.IP.IP.String(), "", conf.GratuitousArpRequest)
}
args.IfName = ifName
result, _ := t020.GetResult(results[0])
return result.Print()
}
25 changes: 1 addition & 24 deletions cni/veth/veth.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,30 +60,7 @@ func addHostRoute(containerIP *net.IPNet, vethHostName string, src string) error
return err
}
s := net.ParseIP(src)
if err = netlink.RouteAdd(&netlink.Route{
LinkIndex: vethHost.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: containerIP,
Gw: nil,
Src: s,
}); err != nil {
if s != nil {
// compatible change for old kernel which does not support src option
if err1 := netlink.RouteAdd(&netlink.Route{
LinkIndex: vethHost.Attrs().Index,
Scope: netlink.SCOPE_LINK,
Dst: containerIP,
Gw: nil,
}); err1 != nil {
return fmt.Errorf("failed to add route '%v dev %v for old linux kernel': %v. With src option err: %v",
containerIP, vethHostName, err1, err)
} else {
return nil
}
}
return fmt.Errorf("failed to add route '%v dev %v src %v': %v", containerIP, vethHostName, s.String(), err)
}
return nil
return utils.AddHostRoute(containerIP, vethHost, s)
}

// #lizard forgives
Expand Down
3 changes: 2 additions & 1 deletion doc/galaxy-config.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ Galaxy support multiple default networks and configures them to pods according t
{"name":"galaxy-flannel", "type":"galaxy-flannel", "delegate":{"type":"galaxy-veth"}, "subnetFile":"/run/flannel
/subnet.env"},
{"name":"galaxy-k8s-vlan", "type":"galaxy-k8s-vlan", "device":"eth1", "default_bridge_name": "br0"},
{"name": "galaxy-k8s-sriov", "type": "galaxy-k8s-sriov", "device": "eth1", "vf_num": 10}
{"name": "galaxy-k8s-sriov", "type": "galaxy-k8s-sriov", "device": "eth1", "vf_num": 10},
{"name":"galaxy-underlay-veth", "type":"galaxy-underlay-veth", "device":"eth1"}
],
"DefaultNetworks": ["galaxy-flannel"],
"ENIIPNetwork": "galaxy-k8s-vlan"
Expand Down
22 changes: 22 additions & 0 deletions doc/supported-cnis.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,25 @@ TKE route ENI CNI is a [Tencent cloud ENI](https://cloud.tencent.com/product/eni
and policy routing on host to connect host veth pair with ENI eth device.

TKE route ENI CNI gets POD IPs either from CNI Args `ipinfos=[{"ip":"192.168.0.68/26","vlan":2,"gateway":"192.168.0.65"}]` or ipam CNI plugin.

## underlay-veth CNI

Underlay-veth CNI is a underlay network plugin which creates a veth pair to connect host network namespace with container.

Compare with Vlan CNI, underlay-veth cni will not create any bridge device and using host route rules for packet forwarding.

Underlay-veth CNI gets POD IPs either from CNI Args `ipinfos=[{"ip":"192.168.0.68/26","vlan":2,"gateway":"192.168.0.65"}]` or ipam CNI plugin.

Underlay-veth CNI supports configuring vlan for POD IPs, and support multi-vlanId in single node.

Compare with vlan CNI, underlay-veth CNI will not create vlan devices for host. The OS administrator should pre-configure vlan devices(with ip address) if needed.

This is the configuration of underlay-veth CNI.

```golang
type NetConf struct {
types.NetConf
// The device which has IDC ip address, eg. eth0
Device string `json:"device"`
}
```
15 changes: 15 additions & 0 deletions e2e/helper/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,21 @@ func SetupDummyDev(ifName, cidr string) error {
return nil
}

func SetupVlanDev(ifName, parent, cidr string, vlanID int) error {
if out, err := Command("ip", "link", "add", "link",parent, "name", ifName, "type", "vlan", "id", fmt.Sprintf("%d", vlanID)).CombinedOutput(); err != nil {
if !strings.HasPrefix(string(out), "RTNETLINK answers: File exists") {
return fmt.Errorf("failed to add link %s: %v, %s", ifName, err, string(out))
}
}
if out, err := Command("ip", "address", "add", cidr, "dev", ifName).CombinedOutput(); err != nil {
return fmt.Errorf("failed to add address %s to %s: %v, %s", cidr, ifName, err, out)
}
if out, err := Command("ip", "link", "set", ifName, "up").CombinedOutput(); err != nil {
return fmt.Errorf("failed to set up %s: %v, %s", ifName, err, out)
}
return nil
}

func IPInfo(cidr string, vlan uint16) (string, error) {
_, ipNet, err := net.ParseCIDR(cidr)
if err != nil {
Expand Down
13 changes: 13 additions & 0 deletions e2e/underlay/veth/veth_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package veth_test

import (
"testing"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

func TestUnderlayVeth(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Underlay-Veth Suite")
}
50 changes: 50 additions & 0 deletions e2e/underlay/veth/veth_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package veth

import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"tkestack.io/galaxy/e2e/helper"
"tkestack.io/galaxy/pkg/utils"
)

var _ = Describe("galaxy-underlay-veth vlan test", func() {
cni := "galaxy-underlay-veth"
ifaceCidr := "192.168.0.66/26"
vlanCidr := "192.168.2.68/26"
containerCidr := "192.168.2.69/26"
containerId := helper.NewContainerId()

AfterEach(func() {
helper.CleanupNetNS()
helper.CleanupIFace("dummy0.2")
helper.CleanupDummy()
})
It("vlan", func() {
netConf := []byte(`{
"name": "myvlan",
"type": "galaxy-underlay-veth",
"device": "dummy0"
}`)
Expect(helper.SetupDummyDev("dummy0", ifaceCidr)).NotTo(HaveOccurred())
Expect(helper.SetupVlanDev("dummy0.2", "dummy0", vlanCidr, 2)).NotTo(HaveOccurred())
argsStr, err := helper.IPInfo(containerCidr, 2)
Expect(err).NotTo(HaveOccurred())
nsPath := helper.CmdAdd(containerId, "", argsStr, cni,
`{"cniVersion":"0.2.0","ip4":{"ip":"192.168.2.69/26","gateway":"192.168.2.65","routes":[{"dst":"0.0.0.0/0"}]},"dns":{}}`, netConf)
_, err = helper.Ping("192.168.2.69")
Expect(err).NotTo(HaveOccurred())

err = (&helper.NetworkTopology{
LeaveDevices: []*helper.LinkDevice{
helper.NewLinkDevice(nil, utils.HostVethName(containerId, ""), "veth"),
},
}).Verify()
Expect(err).NotTo(HaveOccurred())

// check container iface topology, route, neigh, ip address is expected
helper.CheckContainerTopology(nsPath, containerCidr, "192.168.2.65")

// test DEL command
helper.CmdDel(containerId, cni, netConf)
})
})
5 changes: 3 additions & 2 deletions hack/build-native.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@ cd $ROOT
PKG=tkestack.io/galaxy
BIN_PREFIX="galaxy"
# build galaxy cni plugins
PLUGINS="$GOPATH/src/${PKG}/cni/k8s-vlan $GOPATH/src/${PKG}/cni/sdn $GOPATH/src/${PKG}/cni/veth $GOPATH/src/${PKG}/cni/k8s-sriov"
PLUGINS="$GOPATH/src/${PKG}/cni/k8s-vlan $GOPATH/src/${PKG}/cni/sdn $GOPATH/src/${PKG}/cni/veth $GOPATH/src/${PKG}/cni/k8s-sriov $GOPATH/src/${PKG}/cni/underlay/veth"
for d in $PLUGINS; do
if [ -d $d ]; then
plugin=$(basename $d)
plugin_dir=${d#"$GOPATH/src/${PKG}/cni/"}
plugin=${plugin_dir//"/"/"-"}
echo " " $plugin
go build -o bin/${BIN_PREFIX}-$plugin $flags ${PKG}/cni/$plugin
fi
Expand Down
2 changes: 1 addition & 1 deletion hack/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@ echo PATH=$PATH
export GOOS=linux
go test -coverpkg $PKG/pkg/... -coverprofile=coverage.txt -covermode=atomic -v ./tools/... ./cmd/... ./cni/... ./pkg/...
# go tool cover -func=coverage.txt
for i in e2e/k8s-vlan e2e/veth e2e/cni-request; do
for i in e2e/k8s-vlan e2e/veth e2e/cni-request e2e/underlay/veth; do
ginkgo -v $i
done
3 changes: 3 additions & 0 deletions pkg/api/cniutil/cni.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,9 @@ func CmdAdd(cmdArgs *skel.CmdArgs, networkInfos []*NetworkInfo) (types.Result, e
for idx, networkInfo := range networkInfos {
//append additional args from network info
cmdArgs.Args = strings.TrimRight(fmt.Sprintf("%s;%s", cmdArgs.Args, BuildCNIArgs(networkInfo.Args)), ";")
if result != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个改动哪里用到了吗?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不过不用删

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

参考多cni格式,后面改造可能会用到

networkInfo.Conf["prevResult"] = result
}
result, err = DelegateAdd(networkInfo.Conf, cmdArgs, networkInfo.IfName)
if err != nil {
//fail to add cni, then delete all established CNIs recursively
Expand Down
17 changes: 4 additions & 13 deletions pkg/galaxy/galaxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (

"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
glog "k8s.io/klog"
"tkestack.io/galaxy/pkg/api/docker"
Expand Down Expand Up @@ -150,19 +149,11 @@ func (g *Galaxy) Stop() error {
}

func (g *Galaxy) initk8sClient() {
var clientConfig *rest.Config
var err error
if g.Master != "" || g.KubeConf != "" {
clientConfig, err = clientcmd.BuildConfigFromFlags(g.Master, g.KubeConf)
if err != nil {
glog.Fatalf("Invalid client config: %v", err)
}
} else {
clientConfig, err = rest.InClusterConfig()
if err != nil {
glog.Fatalf("Init InClient config failed: %v", err)
}
clientConfig, err := clientcmd.BuildConfigFromFlags(g.Master, g.KubeConf)
if err != nil {
glog.Fatalf("Invalid client config: %v", err)
}

clientConfig.QPS = 1000.0
clientConfig.Burst = 2000
glog.Infof("QPS: %e, Burst: %d", clientConfig.QPS, clientConfig.Burst)
Expand Down
Loading