From 4705ce28e0f1ec62849c5fa4f81c46b5321f42a6 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Tue, 19 Apr 2022 13:18:19 +0800 Subject: [PATCH] windows support for cni server --- .gitignore | 3 +- Makefile | 1 + cmd/daemon/cniserver.go | 11 +- cmd/windows/daemon/main_windows.go | 7 + go.mod | 12 +- go.sum | 48 +++- pkg/daemon/config.go | 98 ++------ pkg/daemon/config_linux.go | 80 +++++++ pkg/daemon/config_windows.go | 58 +++++ pkg/daemon/controller_windows.go | 233 ++++++++++++++++++ pkg/daemon/gateway.go | 45 ---- pkg/daemon/gateway_linux.go | 45 ++++ pkg/daemon/gateway_windows.go | 86 +++++++ pkg/daemon/handler.go | 26 +- pkg/daemon/handler_linux.go | 6 + pkg/daemon/handler_windows.go | 30 +++ pkg/daemon/listen_windows.go | 18 ++ pkg/daemon/ovs_linux.go | 6 +- pkg/daemon/ovs_windows.go | 339 ++++++++++++++++++++++++++ pkg/ovs/ovs-vsctl.go | 360 ---------------------------- pkg/ovs/ovs-vsctl_linux.go | 373 +++++++++++++++++++++++++++++ pkg/ovs/ovs-vsctl_windows.go | 49 ++++ pkg/util/const_linux.go | 3 + pkg/util/const_windows.go | 11 + pkg/util/link_windows.go | 21 ++ pkg/util/network_windows.go | 272 +++++++++++++++++++++ 26 files changed, 1731 insertions(+), 510 deletions(-) create mode 100644 cmd/windows/daemon/main_windows.go create mode 100644 pkg/daemon/config_windows.go create mode 100644 pkg/daemon/controller_windows.go create mode 100644 pkg/daemon/gateway_windows.go create mode 100644 pkg/daemon/handler_windows.go create mode 100644 pkg/daemon/listen_windows.go create mode 100644 pkg/daemon/ovs_windows.go create mode 100644 pkg/ovs/ovs-vsctl_linux.go create mode 100644 pkg/ovs/ovs-vsctl_windows.go create mode 100644 pkg/util/const_windows.go create mode 100644 pkg/util/link_windows.go create mode 100644 pkg/util/network_windows.go diff --git a/.gitignore b/.gitignore index 4f5f52d1c9c..bb4a74e16a8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,9 @@ .vscode/* .DS_Store dist/images/kube-ovn-cmd -dist/images/kube-ovn.exe dist/images/kube-ovn-webhook +dist/images/kube-ovn.exe +dist/images/kube-ovn-daemon.exe test/e2e/ovnnb_db.* test/e2e/ovnsb_db.* kube-ovn.yaml diff --git a/Makefile b/Makefile index 9959a2193d9..7c6b5b526d5 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ build-go: build-go-windows: go mod tidy CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn.exe -ldflags $(GOLDFLAGS) -v ./cmd/windows/cni + CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -buildmode=pie -o $(CURDIR)/dist/images/kube-ovn-daemon.exe -ldflags $(GOLDFLAGS) -v ./cmd/windows/daemon .PHONY: build-go-arm build-go-arm: diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go index 41399916037..1330082e776 100644 --- a/cmd/daemon/cniserver.go +++ b/cmd/daemon/cniserver.go @@ -7,6 +7,7 @@ import ( "net/http" _ "net/http/pprof" // #nosec "os" + "path/filepath" "strings" "time" @@ -75,21 +76,21 @@ func CmdMain() { kubeovnInformerFactory.Start(stopCh) go ctl.Run(stopCh) go daemon.RunServer(config, ctl) - if err := mvCNIConf(config.CniConfName); err != nil { + if err := mvCNIConf(config.CniConfDir, config.CniConfFile, config.CniConfName); err != nil { klog.Fatalf("failed to mv cni conf, %v", err) } http.Handle("/metrics", promhttp.Handler()) klog.Fatal(http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", config.PprofPort), nil)) } -func mvCNIConf(confName string) error { - data, err := os.ReadFile("/kube-ovn/01-kube-ovn.conflist") +func mvCNIConf(configDir, configFile, confName string) error { + data, err := os.ReadFile(configFile) if err != nil { return err } - cniConfPath := fmt.Sprintf("/etc/cni/net.d/%s", confName) - return os.WriteFile(cniConfPath, data, 0444) + cniConfPath := filepath.Join(configDir, confName) + return os.WriteFile(cniConfPath, data, 0644) } func Retry(attempts int, sleep int, f func(configuration *daemon.Configuration) error, ctrl *daemon.Configuration) (err error) { diff --git a/cmd/windows/daemon/main_windows.go b/cmd/windows/daemon/main_windows.go new file mode 100644 index 00000000000..b9d7570f221 --- /dev/null +++ b/cmd/windows/daemon/main_windows.go @@ -0,0 +1,7 @@ +package main + +import "github.com/kubeovn/kube-ovn/cmd/daemon" + +func main() { + daemon.CmdMain() +} diff --git a/go.mod b/go.mod index 1775ad6b012..265d1b578b3 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.17 require ( github.com/Mellanox/sriovnet v1.0.3 github.com/Microsoft/go-winio v0.5.2 + github.com/Microsoft/hcsshim v0.9.2 github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible + github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 github.com/containernetworking/cni v1.0.1 github.com/containernetworking/plugins v1.1.1 @@ -26,7 +28,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/vishvananda/netlink v1.1.1-0.20211101163509-b10eb8fe5cf6 - golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f + golang.org/x/sys v0.0.0-20220412211240-33da011f77ad google.golang.org/grpc v1.40.0 gopkg.in/k8snetworkplumbingwg/multus-cni.v3 v3.7.2 k8s.io/api v0.23.1 @@ -40,12 +42,11 @@ require ( ) require ( - github.com/Microsoft/hcsshim v0.8.22 // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect - github.com/containerd/cgroups v1.0.1 // indirect + github.com/containerd/cgroups v1.0.3 // indirect github.com/coreos/prometheus-operator v0.38.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect @@ -61,8 +62,8 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.5.0 // indirect - github.com/google/go-cmp v0.5.5 // indirect + github.com/golang/mock v1.6.0 // indirect + github.com/google/go-cmp v0.5.6 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect @@ -71,6 +72,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/juju/errors v0.0.0-20220331221717-b38fca44723b // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index bf9dda15f0f..14f292faebb 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,10 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.22 h1:CulZ3GW8sNJExknToo+RWD+U+6ZM5kkNfuxywSDPd08= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0= +github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -177,6 +179,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e h1:KCjb01YiNoRaJ5c+SbnPLWjVzU9vqRYHg3e5JcN50nM= +github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e/go.mod h1:f7vw6ObmmNcyFQLhZX9eUGBJGpnwTJFDvVjqZxIxHWY= github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= @@ -200,6 +204,7 @@ github.com/caddyserver/caddy v1.0.3/go.mod h1:G+ouvOY32gENkJC+jhgl62TyhvqEsFaDiZ github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -253,8 +258,9 @@ github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= @@ -276,6 +282,7 @@ github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -303,11 +310,13 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= @@ -379,12 +388,15 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUn github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -587,8 +599,9 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v0.0.0-20170726212829-748d386b5c1e/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -629,8 +642,10 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -683,6 +698,7 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -769,6 +785,7 @@ github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= @@ -788,6 +805,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/errors v0.0.0-20220331221717-b38fca44723b h1:AxFeSQJfcm2O3ov1wqAkTKYFsnMw2g1B4PkYujfAdkY= +github.com/juju/errors v0.0.0-20220331221717-b38fca44723b/go.mod h1:jMGj9DWF/qbo91ODcfJq6z/RYc3FX3taCBZMCcpI4Ls= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= @@ -814,6 +833,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.0.0-20160823170715-cfb55aafdaf3/go.mod h1:Bvhd+E3laJ0AVkG0c9rmtZcnhV0HQ3+c3YxxqTvc/gA= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= @@ -834,6 +854,7 @@ github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.0/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY= github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA= @@ -874,10 +895,12 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc h1:m7rJJJeXrYCFpsxXYapkDW53wJCDmf9bsIXUg0HoeQY= github.com/mdlayher/arp v0.0.0-20191213142603-f72070a231fc/go.mod h1:eOj1DDj3NAZ6yv+WafaKzY37MFZ58TdfIhQ+8nQbiis= github.com/mdlayher/ethernet v0.0.0-20190313224307-5b5fc417d966/go.mod h1:5s5p/sMJ6sNsFl6uCh85lkFGV8kLuIYJCRJLavVJwvg= @@ -949,7 +972,6 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= github.com/neverlee/keymutex v0.0.0-20171121013845-f593aa834bf9 h1:UfW5pM66x0MWE72ySrpd2Ymrn+b62kNHirozKkY3ojE= github.com/neverlee/keymutex v0.0.0-20171121013845-f593aa834bf9/go.mod h1:3hf2IoUXDKjCg/EuqSLUB5TY8StGS3haWYJiqzP907c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -972,6 +994,7 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= @@ -987,6 +1010,7 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= @@ -1147,6 +1171,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1557,6 +1582,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1644,8 +1670,8 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f h1:rlezHXNlxYWvBCzNses9Dlc7nGFaNMJeqLolcmQSSZY= -golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= @@ -1667,6 +1693,7 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= @@ -1693,6 +1720,7 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190813034749-528a2984e271/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1736,6 +1764,7 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1743,6 +1772,7 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -1831,6 +1861,7 @@ google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1903,8 +1934,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go index 832809e9da1..2ecf4deb990 100644 --- a/pkg/daemon/config.go +++ b/pkg/daemon/config.go @@ -1,8 +1,6 @@ package daemon import ( - "context" - "errors" "flag" "fmt" "net" @@ -13,7 +11,6 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/pflag" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -42,6 +39,8 @@ type Configuration struct { EncapChecksum bool PprofPort int NetworkType string + CniConfDir string + CniConfFile string CniConfName string DefaultProviderName string DefaultInterfaceName string @@ -52,12 +51,13 @@ type Configuration struct { // TODO: validate configuration func ParseFlags(nicBridgeMappings map[string]string) (*Configuration, error) { var ( + argNodeName = pflag.String("node-name", "", "Name of the node on which the daemon is running on.") argIface = pflag.String("iface", "", "The iface used to inter-host pod communication, can be a nic name or a group of regex separated by comma (default the default route iface)") argDPDKTunnelIface = pflag.String("dpdk-tunnel-iface", "br-phy", "Specifies the name of the dpdk tunnel iface.") argMTU = pflag.Int("mtu", 0, "The MTU used by pod iface in overlay networks (default iface MTU - 100)") argEnableMirror = pflag.Bool("enable-mirror", false, "Enable traffic mirror (default false)") argMirrorNic = pflag.String("mirror-iface", "mirror0", "The mirror nic name that will be created by kube-ovn") - argBindSocket = pflag.String("bind-socket", "/run/openvswitch/kube-ovn-daemon.sock", "The socket daemon bind to.") + argBindSocket = pflag.String("bind-socket", defaultBindSocket, "The socket daemon bind to.") argOvsSocket = pflag.String("ovs-socket", "", "The socket to local ovs-server") argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.") argServiceClusterIPRange = pflag.String("service-cluster-ip-range", "10.96.0.0/12", "The kubernetes service cluster ip range") @@ -65,7 +65,9 @@ func ParseFlags(nicBridgeMappings map[string]string) (*Configuration, error) { argEncapChecksum = pflag.Bool("encap-checksum", true, "Enable checksum") argPprofPort = pflag.Int("pprof-port", 10665, "The port to get profiling data") - argsNetworkType = pflag.String("network-type", "geneve", "The ovn network type") + argsNetworkType = pflag.String("network-type", defaultNetworkType, "The ovn network type") + argCniConfDir = pflag.String("cni-conf-dir", util.DefaultCniConfigDir, "Path of the CNI config directory.") + argCniConfFile = pflag.String("cni-conf-file", util.DefaultCniConfigFile, "Path of the CNI config file.") argsCniConfName = pflag.String("cni-conf-name", "01-kube-ovn.conflist", "Specify the name of kube ovn conflist name in dir /etc/cni/net.d/, default: 01-kube-ovn.conflist") argsDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") argsDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") @@ -93,11 +95,18 @@ func ParseFlags(nicBridgeMappings map[string]string) (*Configuration, error) { pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.Parse() - nodeName := os.Getenv(util.HostnameEnv) + nodeName := *argNodeName if nodeName == "" { - klog.Errorf("env KUBE_NODE_NAME not exists") - return nil, fmt.Errorf("env KUBE_NODE_NAME not exists") + klog.Info("node name not specified in command line parameters, fall back to the environment variable") + if nodeName = os.Getenv(util.HostnameEnv); nodeName == "" { + klog.Info("node name not specified in environment variables, fall back to the hostname") + var err error + if nodeName, err = os.Hostname(); err != nil { + return nil, fmt.Errorf("failed to get hostname: %v", err) + } + } } + config := &Configuration{ Iface: *argIface, DPDKTunnelIface: *argDPDKTunnelIface, @@ -108,11 +117,13 @@ func ParseFlags(nicBridgeMappings map[string]string) (*Configuration, error) { OvsSocket: *argOvsSocket, KubeConfigFile: *argKubeConfigFile, PprofPort: *argPprofPort, - NodeName: nodeName, + NodeName: strings.ToLower(nodeName), ServiceClusterIPRange: *argServiceClusterIPRange, NodeLocalDnsIP: *argNodeLocalDnsIP, EncapChecksum: *argEncapChecksum, NetworkType: *argsNetworkType, + CniConfDir: *argCniConfDir, + CniConfFile: *argCniConfFile, CniConfName: *argsCniConfName, DefaultProviderName: *argsDefaultProviderName, DefaultInterfaceName: *argsDefaultInterfaceName, @@ -131,75 +142,6 @@ func ParseFlags(nicBridgeMappings map[string]string) (*Configuration, error) { return config, nil } -func (config *Configuration) initNicConfig(nicBridgeMappings map[string]string) error { - var ( - iface *net.Interface - err error - encapIP string - ) - - // Support to specify node network card separately - node, err := config.KubeClient.CoreV1().Nodes().Get(context.Background(), config.NodeName, metav1.GetOptions{}) - if err != nil { - klog.Errorf("Failed to find node info, err: %v", err) - return err - } - if nodeTunnelName := node.GetAnnotations()[util.TunnelInterfaceAnnotation]; nodeTunnelName != "" { - config.Iface = nodeTunnelName - klog.Infof("Find node tunnel interface name: %v", nodeTunnelName) - } - - isDPDKNode := node.GetLabels()[util.OvsDpTypeLabel] == "userspace" - if isDPDKNode { - config.Iface = config.DPDKTunnelIface - } - if config.Iface == "" { - podIP, ok := os.LookupEnv(util.POD_IP) - if !ok || podIP == "" { - return errors.New("failed to lookup env POD_IP") - } - iface, err = getIfaceOwnPodIP(podIP) - if err != nil { - klog.Errorf("failed to find POD_IP iface %v", err) - return err - } - encapIP = podIP - } else { - tunnelNic := config.Iface - if brName := nicBridgeMappings[tunnelNic]; brName != "" { - klog.Infof("nic %s has been bridged to %s, use %s as the tunnel interface instead", tunnelNic, brName, brName) - tunnelNic = brName - } - - iface, err = findInterface(tunnelNic) - if err != nil { - klog.Errorf("failed to find iface %s, %v", tunnelNic, err) - return err - } - addrs, err := iface.Addrs() - if err != nil { - return fmt.Errorf("failed to get iface addr. %v", err) - } - if len(addrs) == 0 { - return fmt.Errorf("iface %s has no ip address", tunnelNic) - } - encapIP = strings.Split(addrs[0].String(), "/")[0] - } - - if config.MTU == 0 { - config.MTU = iface.MTU - util.GeneveHeaderLength - } - - config.MSS = config.MTU - util.TcpIpHeaderLength - if !config.EncapChecksum { - if err := disableChecksum(); err != nil { - klog.Errorf("failed to set checksum offload, %v", err) - } - } - - return setEncapIP(encapIP) -} - func findInterface(ifaceStr string) (*net.Interface, error) { iface, err := net.InterfaceByName(ifaceStr) if err == nil && iface != nil { diff --git a/pkg/daemon/config_linux.go b/pkg/daemon/config_linux.go index 33f2338bbd9..97640a88bab 100644 --- a/pkg/daemon/config_linux.go +++ b/pkg/daemon/config_linux.go @@ -1,13 +1,93 @@ package daemon import ( + "context" "errors" "fmt" "net" + "os" + "strings" + "github.com/kubeovn/kube-ovn/pkg/util" "github.com/vishvananda/netlink" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" ) +const ( + defaultBindSocket = `/run/openvswitch/kube-ovn-daemon.sock` + defaultNetworkType = `geneve` +) + +func (config *Configuration) initNicConfig(nicBridgeMappings map[string]string) error { + var ( + iface *net.Interface + err error + encapIP string + ) + + // Support to specify node network card separately + node, err := config.KubeClient.CoreV1().Nodes().Get(context.Background(), config.NodeName, metav1.GetOptions{}) + if err != nil { + klog.Errorf("Failed to find node info, err: %v", err) + return err + } + if nodeTunnelName := node.GetAnnotations()[util.TunnelInterfaceAnnotation]; nodeTunnelName != "" { + config.Iface = nodeTunnelName + klog.Infof("Find node tunnel interface name: %v", nodeTunnelName) + } + + isDPDKNode := node.GetLabels()[util.OvsDpTypeLabel] == "userspace" + if isDPDKNode { + config.Iface = config.DPDKTunnelIface + } + if config.Iface == "" { + podIP, ok := os.LookupEnv(util.POD_IP) + if !ok || podIP == "" { + return errors.New("failed to lookup env POD_IP") + } + iface, err = getIfaceOwnPodIP(podIP) + if err != nil { + klog.Errorf("failed to find POD_IP iface %v", err) + return err + } + encapIP = podIP + } else { + tunnelNic := config.Iface + if brName := nicBridgeMappings[tunnelNic]; brName != "" { + klog.Infof("nic %s has been bridged to %s, use %s as the tunnel interface instead", tunnelNic, brName, brName) + tunnelNic = brName + } + + iface, err = findInterface(tunnelNic) + if err != nil { + klog.Errorf("failed to find iface %s, %v", tunnelNic, err) + return err + } + addrs, err := iface.Addrs() + if err != nil { + return fmt.Errorf("failed to get iface addr. %v", err) + } + if len(addrs) == 0 { + return fmt.Errorf("iface %s has no ip address", tunnelNic) + } + encapIP = strings.Split(addrs[0].String(), "/")[0] + } + + if config.MTU == 0 { + config.MTU = iface.MTU - util.GeneveHeaderLength + } + + config.MSS = config.MTU - util.TcpIpHeaderLength + if !config.EncapChecksum { + if err := disableChecksum(); err != nil { + klog.Errorf("failed to set checksum offload, %v", err) + } + } + + return setEncapIP(encapIP) +} + func getIfaceOwnPodIP(podIP string) (*net.Interface, error) { links, err := netlink.LinkList() if err != nil { diff --git a/pkg/daemon/config_windows.go b/pkg/daemon/config_windows.go new file mode 100644 index 00000000000..a0edbd3c1ee --- /dev/null +++ b/pkg/daemon/config_windows.go @@ -0,0 +1,58 @@ +package daemon + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/util" +) + +const ( + defaultBindSocket = util.WindowsListenPipe + defaultNetworkType = `vxlan` +) + +func (config *Configuration) initNicConfig(nicBridgeMappings map[string]string) error { + node, err := config.KubeClient.CoreV1().Nodes().Get(context.Background(), config.NodeName, metav1.GetOptions{}) + if err != nil { + klog.Errorf("Failed to get node %s: %v", config.NodeName, err) + return err + } + + nodeIP, _ := util.GetNodeInternalIP(*node) + if nodeIP == "" { + klog.Errorf("failed to get internal IPv4 for node %s", config.NodeName) + return err + } + iface, mtu, err := getIfaceByIP(nodeIP) + if err != nil { + klog.Errorf("failed to get interface by IP %s: %v", nodeIP, err) + return err + } + + config.Iface = iface + if config.MTU == 0 { + config.MTU = mtu - util.GeneveHeaderLength + } + + config.MSS = config.MTU - util.TcpIpHeaderLength + if !config.EncapChecksum { + if err := disableChecksum(); err != nil { + klog.Errorf("failed to set checksum offload, %v", err) + } + } + + return setEncapIP(nodeIP) +} + +func getIfaceByIP(ip string) (string, int, error) { + iface, err := util.GetInterfaceByIP(ip) + if err != nil { + klog.Error(err) + return "", 0, err + } + + return iface.InterfaceAlias, int(iface.NlMtu), err +} diff --git a/pkg/daemon/controller_windows.go b/pkg/daemon/controller_windows.go new file mode 100644 index 00000000000..20df6ff647d --- /dev/null +++ b/pkg/daemon/controller_windows.go @@ -0,0 +1,233 @@ +package daemon + +import ( + "fmt" + "net" + "strings" + + v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// ControllerRuntime represents runtime specific controller members +type ControllerRuntime struct { +} + +func (c *Controller) initRuntime() error { + return nil +} + +func (c *Controller) reconcileRouters(_ subnetEvent) error { + klog.Info("reconcile routes") + node, err := c.nodesLister.Get(c.config.NodeName) + if err != nil { + klog.Errorf("failed to get node %s %v", c.config.NodeName, err) + return err + } + gateway, ok := node.Annotations[util.GatewayAnnotation] + if !ok { + klog.Errorf("annotation for node %s ovn.kubernetes.io/gateway not exists", node.Name) + return fmt.Errorf("annotation for node ovn.kubernetes.io/gateway not exists") + } + + subnets, err := c.subnetsLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list subnets: %v", err) + return err + } + + gwIPv4, gwIPv6 := util.SplitStringIP(gateway) + v4Cidrs, v6Cidrs := make([]string, 0, len(subnets)), make([]string, 0, len(subnets)) + for _, subnet := range subnets { + if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || + subnet.Spec.Vpc != util.DefaultVpc || + !subnet.Status.IsReady() { + continue + } + + for _, cidrBlock := range strings.Split(subnet.Spec.CIDRBlock, ",") { + if _, ipNet, err := net.ParseCIDR(cidrBlock); err != nil { + klog.Errorf("%s is not a valid cidr block", cidrBlock) + } else { + _, bits := ipNet.Mask.Size() + if bits == 32 { + if gwIPv4 != "" && !util.CIDRContainIP(cidrBlock, gwIPv4) { + v4Cidrs = append(v4Cidrs, ipNet.String()) + } + } else { + if gwIPv6 != "" && !util.CIDRContainIP(cidrBlock, gwIPv6) { + v6Cidrs = append(v6Cidrs, ipNet.String()) + } + } + } + } + } + + adapter, err := util.GetNetAdapter(util.NodeNic, false) + if err != nil { + klog.Errorf("failed to get network adapter %s: %v", util.NodeNic, err) + return err + } + routes, err := util.GetNetRoute(adapter.InterfaceIndex) + if err != nil { + klog.Errorf("failed to get NetIPRoute with index %d: %v", adapter.InterfaceIndex, err) + return err + } + + existingRoutes := make([]string, 0, len(routes)) + for _, route := range routes { + if route.NextHop == "0.0.0.0" || route.NextHop == "::" { + continue + } + existingRoutes = append(existingRoutes, route.DestinationPrefix) + } + + toAddV4, toAddV6, toDel := routeDiff(existingRoutes, v4Cidrs, v6Cidrs) + klog.Infof("routes to be added: %v", append(toAddV4, toAddV6...)) + klog.Infof("routes to be removed: %v", toDel) + for _, r := range toAddV4 { + if err = util.NewNetRoute(adapter.InterfaceIndex, r, gwIPv4); err != nil { + klog.Errorf("failed to del route %s: %v", r, err) + } + } + for _, r := range toAddV6 { + if err = util.NewNetRoute(adapter.InterfaceIndex, r, gwIPv6); err != nil { + klog.Errorf("failed to del route %s: %v", r, err) + } + } + for _, r := range toDel { + if err = util.RemoveNetRoute(adapter.InterfaceIndex, r); err != nil { + klog.Errorf("failed to remove route %s: %v", r, err) + } + } + + return nil +} + +func routeDiff(existingRoutes, v4Cidrs, v6Cidrs []string) (toAddV4, toAddV6, toDel []string) { + existing := make(map[string]struct{}, len(existingRoutes)) + expectedV4 := make(map[string]struct{}, len(v4Cidrs)) + expectedV6 := make(map[string]struct{}, len(v6Cidrs)) + for _, r := range existingRoutes { + existing[r] = struct{}{} + } + + var ok bool + for _, r := range v4Cidrs { + expectedV4[r] = struct{}{} + if _, ok = existing[r]; !ok { + toAddV4 = append(toAddV4, r) + } + } + for _, r := range v6Cidrs { + expectedV6[r] = struct{}{} + if _, ok = existing[r]; !ok { + toAddV6 = append(toAddV6, r) + } + } + for _, r := range existingRoutes { + if _, ok = expectedV4[r]; ok { + continue + } + if _, ok = expectedV6[r]; ok { + continue + } + toDel = append(toDel, r) + } + + return +} + +func (c *Controller) handlePod(key string) error { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + klog.Infof("handle pod %s/%s", namespace, name) + + pod, err := c.podsLister.Pods(namespace).Get(name) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + + if err := util.ValidatePodNetwork(pod.Annotations); err != nil { + klog.Errorf("validate pod %s/%s failed, %v", namespace, name, err) + c.recorder.Eventf(pod, v1.EventTypeWarning, "ValidatePodNetworkFailed", err.Error()) + return err + } + + podName := pod.Name + if pod.Annotations[fmt.Sprintf(util.VmTemplate, util.OvnProvider)] != "" { + podName = pod.Annotations[fmt.Sprintf(util.VmTemplate, util.OvnProvider)] + } + + // set default nic bandwidth + ifaceID := ovs.PodNameToPortName(podName, pod.Namespace, util.OvnProvider) + err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[util.EgressRateAnnotation], pod.Annotations[util.IngressRateAnnotation], pod.Annotations[util.PriorityAnnotation]) + if err != nil { + return err + } + err = ovs.ConfigInterfaceMirror(c.config.EnableMirror, pod.Annotations[util.MirrorControlAnnotation], ifaceID) + if err != nil { + return err + } + + // set multus-nic bandwidth + attachNets, err := util.ParsePodNetworkAnnotation(pod.Annotations[util.AttachmentNetworkAnnotation], pod.Namespace) + if err != nil { + return err + } + for _, multiNet := range attachNets { + provider := fmt.Sprintf("%s.%s.ovn", multiNet.Name, multiNet.Namespace) + if pod.Annotations[fmt.Sprintf(util.VmTemplate, provider)] != "" { + podName = pod.Annotations[fmt.Sprintf(util.VmTemplate, provider)] + } + if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" { + ifaceID = ovs.PodNameToPortName(podName, pod.Namespace, provider) + err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[fmt.Sprintf(util.EgressRateAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.IngressRateAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.PriorityAnnotationTemplate, provider)]) + if err != nil { + return err + } + err = ovs.ConfigInterfaceMirror(c.config.EnableMirror, pod.Annotations[fmt.Sprintf(util.MirrorControlAnnotationTemplate, provider)], ifaceID) + if err != nil { + return err + } + err = ovs.SetNetemQos(podName, pod.Namespace, ifaceID, pod.Annotations[fmt.Sprintf(util.NetemQosLatencyAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.NetemQosLimitAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.NetemQosLossAnnotationTemplate, provider)]) + if err != nil { + return err + } + } + } + return nil +} + +func (c *Controller) loopEncapIpCheck() { + // TODO +} + +func (c *Controller) loopCheckSubnetQosPriority() { + // TODO +} + +func (c *Controller) clearQos(podName, podNamespace, ifaceID string) error { + // TODO + return nil +} + +func rotateLog() { + // TODO +} + +func (c *Controller) operateMod() { +} diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go index 7cdeb6ceabc..3128fde79ea 100644 --- a/pkg/daemon/gateway.go +++ b/pkg/daemon/gateway.go @@ -7,7 +7,6 @@ import ( "strings" v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" @@ -40,50 +39,6 @@ func (c *Controller) runGateway() { c.appendMssRule() } -func (c *Controller) addEgressConfig(subnet *kubeovnv1.Subnet, ip string) error { - if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || - subnet.Spec.GatewayType != kubeovnv1.GWDistributedType || - subnet.Spec.Vpc != util.DefaultVpc { - return nil - } - - if !subnet.Spec.NatOutgoing && subnet.Spec.ExternalEgressGateway != "" { - podIPs := strings.Split(ip, ",") - protocol := util.CheckProtocol(ip) - return c.addPodPolicyRouting(protocol, subnet.Spec.ExternalEgressGateway, subnet.Spec.PolicyRoutingPriority, subnet.Spec.PolicyRoutingTableID, podIPs) - } - - return nil -} - -func (c *Controller) removeEgressConfig(subnet, ip string) error { - if subnet == "" || ip == "" { - return nil - } - - podSubnet, err := c.subnetsLister.Get(subnet) - if k8serrors.IsNotFound(err) { - return nil - } else if err != nil { - klog.Errorf("failed to get subnet %s: %+v", subnet, err) - return err - } - - if (podSubnet.Spec.Vlan != "" && !podSubnet.Spec.LogicalGateway) || - podSubnet.Spec.GatewayType != kubeovnv1.GWDistributedType || - podSubnet.Spec.Vpc != util.DefaultVpc { - return nil - } - - if !podSubnet.Spec.NatOutgoing && podSubnet.Spec.ExternalEgressGateway != "" { - podIPs := strings.Split(ip, ",") - protocol := util.CheckProtocol(ip) - return c.deletePodPolicyRouting(protocol, podSubnet.Spec.ExternalEgressGateway, podSubnet.Spec.PolicyRoutingPriority, podSubnet.Spec.PolicyRoutingTableID, podIPs) - } - - return nil -} - func (c *Controller) setGatewayBandwidth() error { node, err := c.nodesLister.Get(c.config.NodeName) if err != nil { diff --git a/pkg/daemon/gateway_linux.go b/pkg/daemon/gateway_linux.go index a42b7c636fa..58978e0d81a 100644 --- a/pkg/daemon/gateway_linux.go +++ b/pkg/daemon/gateway_linux.go @@ -12,6 +12,7 @@ import ( "github.com/alauda/felix/ipsets" "github.com/vishvananda/netlink" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" @@ -447,6 +448,50 @@ func (c *Controller) setIptables() error { return nil } +func (c *Controller) addEgressConfig(subnet *kubeovnv1.Subnet, ip string) error { + if (subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway) || + subnet.Spec.GatewayType != kubeovnv1.GWDistributedType || + subnet.Spec.Vpc != util.DefaultVpc { + return nil + } + + if !subnet.Spec.NatOutgoing && subnet.Spec.ExternalEgressGateway != "" { + podIPs := strings.Split(ip, ",") + protocol := util.CheckProtocol(ip) + return c.addPodPolicyRouting(protocol, subnet.Spec.ExternalEgressGateway, subnet.Spec.PolicyRoutingPriority, subnet.Spec.PolicyRoutingTableID, podIPs) + } + + return nil +} + +func (c *Controller) removeEgressConfig(subnet, ip string) error { + if subnet == "" || ip == "" { + return nil + } + + podSubnet, err := c.subnetsLister.Get(subnet) + if k8serrors.IsNotFound(err) { + return nil + } else if err != nil { + klog.Errorf("failed to get subnet %s: %+v", subnet, err) + return err + } + + if (podSubnet.Spec.Vlan != "" && !podSubnet.Spec.LogicalGateway) || + podSubnet.Spec.GatewayType != kubeovnv1.GWDistributedType || + podSubnet.Spec.Vpc != util.DefaultVpc { + return nil + } + + if !podSubnet.Spec.NatOutgoing && podSubnet.Spec.ExternalEgressGateway != "" { + podIPs := strings.Split(ip, ",") + protocol := util.CheckProtocol(ip) + return c.deletePodPolicyRouting(protocol, podSubnet.Spec.ExternalEgressGateway, podSubnet.Spec.PolicyRoutingPriority, podSubnet.Spec.PolicyRoutingTableID, podIPs) + } + + return nil +} + func (c *Controller) setExGateway() error { node, err := c.nodesLister.Get(c.config.NodeName) if err != nil { diff --git a/pkg/daemon/gateway_windows.go b/pkg/daemon/gateway_windows.go new file mode 100644 index 00000000000..b09aa3daecb --- /dev/null +++ b/pkg/daemon/gateway_windows.go @@ -0,0 +1,86 @@ +package daemon + +import ( + "context" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c *Controller) setIPSet() error { + return nil +} + +func (c *Controller) setPolicyRouting() error { + return nil +} + +func (c *Controller) setIptables() error { + return nil +} + +func (c *Controller) addEgressConfig(subnet *kubeovnv1.Subnet, ip string) error { + // nothing to do on Windows + return nil +} + +func (c *Controller) removeEgressConfig(subnet, ip string) error { + // nothing to do on Windows + return nil +} + +func (c *Controller) setExGateway() error { + node, err := c.nodesLister.Get(c.config.NodeName) + if err != nil { + klog.Errorf("failed to get node, %v", err) + return err + } + enable := node.Labels[util.ExGatewayLabel] + if enable == "true" { + cm, err := c.config.KubeClient.CoreV1().ConfigMaps(c.config.ExternalGatewayConfigNS).Get(context.Background(), util.ExternalGatewayConfig, metav1.GetOptions{}) + if err != nil { + klog.Errorf("failed to get ovn-external-gw-config, %v", err) + return err + } + // enable external-gw-config without 'external-gw-nic' configured + // to reuse existing physical network from arg 'external-gateway-net' + // TODO + if _, err := ovs.Exec( + ovs.MayExist, "add-br", "br-external", "--", + ovs.MayExist, "add-port", "br-external", cm.Data["external-gw-nic"], + ); err != nil { + return fmt.Errorf("failed to enable external gateway, %v", err) + } + + output, err := ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings") + if err != nil { + return fmt.Errorf("failed to get external-ids, %v", err) + } + bridgeMappings := "external:br-external" + if output != "" && !util.IsStringIn(bridgeMappings, strings.Split(output, ",")) { + bridgeMappings = fmt.Sprintf("%s,%s", output, bridgeMappings) + } + + output, err = ovs.Exec("set", "open", ".", fmt.Sprintf("external-ids:ovn-bridge-mappings=%s", bridgeMappings)) + if err != nil { + return fmt.Errorf("failed to set bridge-mappings, %v: %q", err, output) + } + } else { + if _, err := ovs.Exec( + ovs.IfExists, "del-br", "br-external"); err != nil { + return fmt.Errorf("failed to disable external gateway, %v", err) + } + } + return nil +} + +//Generally, the MTU of the interface is set to 1400. But in special cases, a special pod (docker indocker) will introduce the docker0 interface to the pod. The MTU of docker0 is 1500. +//The network application in pod will calculate the TCP MSS according to the MTU of docker0, and then initiate communication with others. After the other party sends a response, the kernel protocol stack of Linux host will send ICMP unreachable message to the other party, indicating that IP fragmentation is needed, which is not supported by the other party, resulting in communication failure. +func (c *Controller) appendMssRule() { +} diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go index 6b65606243a..c2b5627491e 100644 --- a/pkg/daemon/handler.go +++ b/pkg/daemon/handler.go @@ -74,7 +74,15 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon return } - klog.Infof("add port request %v", podRequest) + klog.Infof("add port request: %v", podRequest) + if err := csh.validatePodRequest(&podRequest); err != nil { + klog.Error(err) + if err := resp.WriteHeaderAndEntity(http.StatusBadRequest, request.CniResponse{Err: err.Error()}); err != nil { + klog.Errorf("failed to write response, %v", err) + } + return + } + var gatewayCheckMode int var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, providerNetwork, ifName, nicType, podNicName, priority, vmName string var isDefaultRoute bool @@ -231,12 +239,12 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, podRequest.Routes) if nicType == util.InternalType { - podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode) + podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode) } else if nicType == util.DpdkType { err = csh.configureDpdkNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, ingress, egress, priority, getShortSharedDir(pod.UID, podRequest.VhostUserSocketVolumeName), podRequest.VhostUserSocketName) } else { podNicName = ifName - err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode) + err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode) } if err != nil { errMsg := fmt.Errorf("configure nic failed %v", err) @@ -356,7 +364,15 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon } } - klog.Infof("delete port request %v", podRequest) + klog.Infof("del port request: %v", podRequest) + if err := csh.validatePodRequest(&podRequest); err != nil { + klog.Error(err) + if err := resp.WriteHeaderAndEntity(http.StatusBadRequest, request.CniResponse{Err: err.Error()}); err != nil { + klog.Errorf("failed to write response, %v", err) + } + return + } + if pod.Annotations != nil && (podRequest.Provider == util.OvnProvider || podRequest.CniType == util.CniTypeName) { subnet := pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, podRequest.Provider)] if subnet != "" { @@ -392,7 +408,7 @@ func (csh cniServerHandler) handleDel(req *restful.Request, resp *restful.Respon podRequest.PodName = vmName } - err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.DeviceID, podRequest.IfName, nicType) + err = csh.deleteNic(podRequest.PodName, podRequest.PodNamespace, podRequest.ContainerID, podRequest.NetNs, podRequest.DeviceID, podRequest.IfName, nicType) if err != nil { errMsg := fmt.Errorf("del nic failed %v", err) klog.Error(errMsg) diff --git a/pkg/daemon/handler_linux.go b/pkg/daemon/handler_linux.go index 1ed56b625cf..d5fe0456b97 100644 --- a/pkg/daemon/handler_linux.go +++ b/pkg/daemon/handler_linux.go @@ -9,9 +9,15 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + "github.com/kubeovn/kube-ovn/pkg/request" "github.com/kubeovn/kube-ovn/pkg/util" ) +func (csh cniServerHandler) validatePodRequest(req *request.CniRequest) error { + // nothing to do on linux + return nil +} + func createShortSharedDir(pod *v1.Pod, volumeName string) (err error) { var volume *v1.Volume for index, v := range pod.Spec.Volumes { diff --git a/pkg/daemon/handler_windows.go b/pkg/daemon/handler_windows.go new file mode 100644 index 00000000000..fe718326e56 --- /dev/null +++ b/pkg/daemon/handler_windows.go @@ -0,0 +1,30 @@ +package daemon + +import ( + "errors" + + v1 "k8s.io/api/core/v1" + + "github.com/kubeovn/kube-ovn/pkg/request" +) + +func (csh cniServerHandler) validatePodRequest(req *request.CniRequest) error { + if req.DeviceID != "" { + return errors.New("SR-IOV is not supported on Windows") + } + if req.VhostUserSocketVolumeName != "" { + return errors.New("DPDK is not supported on Windows") + } + + return nil +} + +func createShortSharedDir(pod *v1.Pod, volumeName string) error { + // nothing to do on Windows + return nil +} + +func removeShortSharedDir(pod *v1.Pod, volumeName string) error { + // nothing to do on Windows + return nil +} diff --git a/pkg/daemon/listen_windows.go b/pkg/daemon/listen_windows.go new file mode 100644 index 00000000000..3e6ccf9f4bd --- /dev/null +++ b/pkg/daemon/listen_windows.go @@ -0,0 +1,18 @@ +package daemon + +import ( + "net" + + "github.com/Microsoft/go-winio" + "k8s.io/klog/v2" +) + +func listen(socket string) (net.Listener, func(), error) { + listener, err := winio.ListenPipe(socket, nil) + if err != nil { + klog.Errorf("failed to listen pipe %s: %v", socket, err) + return nil, nil, err + } + + return listener, func() {}, nil +} diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 995d9c46a87..047cf07a69d 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -55,7 +55,7 @@ func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, ne return nil } -func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) error { +func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) error { var err error var hostNicName, containerNicName string if DeviceID == "" { @@ -122,7 +122,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, return nil } -func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, deviceID, ifName, nicType string) error { +func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, netns, deviceID, ifName, nicType string) error { var nicName string hostNicName, containerNicName := generateNicName(containerID, ifName) @@ -835,7 +835,7 @@ func renameLink(curName, newName string) error { return nil } -func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) (string, error) { +func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) (string, error) { _, containerNicName := generateNicName(containerID, ifName) ipStr := util.GetIpWithoutMask(ip) ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider) diff --git a/pkg/daemon/ovs_windows.go b/pkg/daemon/ovs_windows.go new file mode 100644 index 00000000000..949ada67060 --- /dev/null +++ b/pkg/daemon/ovs_windows.go @@ -0,0 +1,339 @@ +package daemon + +import ( + "errors" + "fmt" + "net" + "strings" + "time" + + "github.com/Microsoft/hcsshim" + "github.com/containernetworking/plugins/pkg/hns" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/request" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway, ingress, egress, priority, sharedDir, socketName string) error { + return errors.New("DPDK is not supported on Windows") +} + +func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) (string, error) { + return ifName, csh.configureNic(podName, podNamespace, provider, netns, containerID, "", ifName, mac, mtu, ip, gateway, isDefaultRoute, routes, dnsServer, dnsSuffix, ingress, egress, priority, DeviceID, nicType, gwCheckMode) +} + +func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) error { + if DeviceID != "" { + return errors.New("SR-IOV is not supported on Windows") + } + + hnsNetwork, err := hcsshim.GetHNSNetworkByName(util.HnsNetwork) + if err != nil { + klog.Errorf("failed to get HNS network %s: %v", util.HnsNetwork) + return err + } + if hnsNetwork == nil { + err = fmt.Errorf("HNS network %s does not exist", util.HnsNetwork) + klog.Error(err) + return err + } + if !strings.EqualFold(hnsNetwork.Type, "Transparent") { + err = fmt.Errorf(`type of HNS network %s is "%s", while "Transparent" is required`, util.HnsNetwork, hnsNetwork.Type) + klog.Error(err) + return err + } + + ipAddr := util.GetIpWithoutMask(ip) + sanbox := hns.GetSandboxContainerID(containerID, netns) + epName := sanbox[:12] + _, err = hns.AddHnsEndpoint(epName, hnsNetwork.Id, containerID, netns, func() (*hcsshim.HNSEndpoint, error) { + ipv4, ipv6 := util.SplitStringIP(ipAddr) + endpoint := &hcsshim.HNSEndpoint{ + Name: epName, + VirtualNetwork: hnsNetwork.Id, + DNSServerList: strings.Join(dnsServer, ","), + DNSSuffix: strings.Join(dnsSuffix, ","), + MacAddress: mac, + IPAddress: net.ParseIP(ipv4), + IPv6Address: net.ParseIP(ipv6), + } + + endpoint.GatewayAddress, endpoint.GatewayAddressV6 = util.SplitStringIP(gateway) + for _, s := range strings.Split(ip, ",") { + _, network, err := net.ParseCIDR(s) + if err != nil { + return nil, err + } + if ones, bits := network.Mask.Size(); bits == 128 { + endpoint.IPv6PrefixLength = uint8(ones) + } else { + endpoint.PrefixLength = uint8(ones) + } + } + + return endpoint, nil + }) + if err != nil { + klog.Errorf("failed to add HNS endpoint: %v", err) + return err + } + + if containerID != sanbox { + // pause container, return here + return nil + } + + // add OVS port + exists, err := ovs.PortExists(epName) + if err != nil { + klog.Error(err) + return err + } + if exists { + return nil + } + + timeout := 5 + adapterName := fmt.Sprintf("vEthernet (%s)", epName) + for i := 0; i < timeout; i++ { + adapter, _ := util.GetNetAdapter(adapterName, true) + if adapter == nil { + time.Sleep(time.Second) + continue + } + + _mtu := uint32(mtu) + if err = util.SetNetIPInterface(adapter.InterfaceIndex, nil, &_mtu, nil, nil); err != nil { + klog.Errorf("failed to set MTU of %s to %d: %v", adapterName, mtu, err) + return err + } + + ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider) + ovs.CleanDuplicatePort(ifaceID, epName) + output, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", epName, "--", + "set", "interface", epName, "type=internal", "--", + "set", "interface", epName, fmt.Sprintf("external_ids:iface-id=%s", ifaceID), + fmt.Sprintf("external_ids:pod_name=%s", podName), + fmt.Sprintf("external_ids:pod_namespace=%s", podNamespace), + fmt.Sprintf("external_ids:ip=%s", ipAddr)) + if err != nil { + return fmt.Errorf("failed to add OVS port %s, %v: %q", epName, err, output) + } + + if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, egress, ingress, priority); err != nil { + return err + } + + return nil + } + + return fmt.Errorf(`failed to get network adapter "%s" after %d seconds`, adapterName, timeout) +} + +func configureNic(name, ip string, mac net.HardwareAddr, mtu int) error { + adapter, err := util.GetNetAdapter(name, false) + if err != nil { + klog.Errorf("failed to get network adapter %s: %v", name, err) + return err + } + + // we need to set mac address before enabling the adapter + if newMac := mac.String(); !strings.EqualFold(newMac, adapter.MacAddress) { + if err = util.SetAdapterMac(name, mac.String()); err != nil { + klog.Error(err) + return err + } + } + + if adapter.InterfaceAdminStatus != 1 { + if err = util.EnableAdapter(name); err != nil { + klog.Error(err) + return err + } + } + + interfaces, err := util.GetNetIPInterface(adapter.InterfaceIndex) + if err != nil { + klog.Errorf("failed to get NetIPInterface with index %d: %v", adapter.InterfaceIndex, err) + return err + } + addresses, err := util.GetNetIPAddress(adapter.InterfaceIndex) + if err != nil { + klog.Errorf("failed to get NetIPAddress with index %d: %v", adapter.InterfaceIndex, err) + return err + } + + // set MTU + for _, iface := range interfaces { + if uint32(mtu) != iface.NlMtu { + _mtu := uint32(mtu) + if err = util.SetNetIPInterface(iface.InterfaceIndex, &iface.AddressFamily, &_mtu, nil, nil); err != nil { + klog.Error(err) + return err + } + } + } + + addrToAdd := make(map[string]interface{}) + for _, addr := range strings.Split(ip, ",") { + addrToAdd[addr] = true + } + + addrToDel := make(map[string]interface{}) + for _, addr := range addresses { + // handle IPv6 address, e.g. fe80::e053:1757:f000:be40%47 + addr.IPAddress = strings.TrimSuffix(addr.IPAddress, fmt.Sprintf("%%%d", addr.InterfaceIndex)) + ip := net.ParseIP(addr.IPAddress) + if ip == nil { + klog.Warningf("found invalid IP address %s on interface %s", addr.IPAddress, name) + continue + } + if ip.IsLinkLocalUnicast() { + // skip 169.254.0.0/16 and fe80::/10 + continue + } + + s := fmt.Sprintf("%s/%d", addr.IPAddress, addr.PrefixLength) + if _, ok := addrToAdd[s]; ok { + delete(addrToAdd, s) + } else { + addrToDel[s] = true + } + } + + for addr := range addrToDel { + if err = util.RemoveNetIPAddress(adapter.InterfaceIndex, addr); err != nil { + return err + } + } + for addr := range addrToAdd { + if err = util.NewNetIPAddress(adapter.InterfaceIndex, addr); err != nil { + return err + } + } + + return nil +} + +func (csh cniServerHandler) deleteNic(podName, podNamespace, containerID, netns, deviceID, ifName, nicType string) error { + epName := hns.ConstructEndpointName(containerID, netns, util.HnsNetwork)[:12] + // remove ovs port + output, err := ovs.Exec(ovs.IfExists, "--with-iface", "del-port", "br-int", epName) + if err != nil { + return fmt.Errorf("failed to delete ovs port %s: %v, %q", epName, err, output) + } + + return hns.RemoveHnsEndpoint(epName, netns, containerID) +} + +func generateNicName(containerID, ifname string) (string, string) { + if ifname == "eth0" { + return fmt.Sprintf("%s_h", containerID[0:12]), fmt.Sprintf("%s_c", containerID[0:12]) + } + return fmt.Sprintf("%s_%s_h", containerID[0:12-len(ifname)], ifname), fmt.Sprintf("%s_%s_c", containerID[0:12-len(ifname)], ifname) +} + +func waitNetworkReady(nic, ipAddr, gateway string, underlayGateway, verbose bool) error { + ips := strings.Split(ipAddr, ",") + for i, gw := range strings.Split(gateway, ",") { + src := strings.Split(ips[i], "/")[0] + if !underlayGateway || util.CheckProtocol(gw) == kubeovnv1.ProtocolIPv6 { + if err := pingGateway(gw, src, verbose); err != nil { + return err + } + } + } + return nil +} + +func configureNodeNic(portName, ip, gw string, macAddr net.HardwareAddr, mtu int) error { + ipStr := util.GetIpWithoutMask(ip) + raw, err := ovs.Exec(ovs.MayExist, "add-port", "br-int", util.NodeNic, "--", + "set", "interface", util.NodeNic, "type=internal", "--", + "set", "interface", util.NodeNic, fmt.Sprintf("external_ids:iface-id=%s", portName), + fmt.Sprintf("external_ids:ip=%s", ipStr)) + if err != nil { + klog.Errorf("failed to configure node nic %s: %v, %q", portName, err, raw) + return fmt.Errorf(raw) + } + + if err = configureNic(util.NodeNic, ip, macAddr, mtu); err != nil { + return err + } + + // ping ovn0 gw to activate the flow + if err := waitNetworkReady(util.NodeNic, ip, gw, false, true); err != nil { + klog.Errorf("failed to init ovn0 check: %v", err) + return err + } + + return nil +} + +// If OVS restart, the ovn0 port will down and prevent host to pod network, +// Restart the kube-ovn-cni when this happens +func (c *Controller) loopOvn0Check() { + // no need to check ovn0 on Windows +} + +func configureMirrorLink(portName string, mtu int) error { + adapter, err := util.GetNetAdapter(portName, false) + if err != nil { + klog.Errorf("failed to get network adapter %s: %v", portName, err) + return err + } + + if adapter.InterfaceAdminStatus != 1 { + if err = util.EnableAdapter(portName); err != nil { + klog.Error(err) + return err + } + } + + interfaces, err := util.GetNetIPInterface(adapter.InterfaceIndex) + if err != nil { + klog.Errorf("failed to get NetIPInterface with index %d: %v", adapter.InterfaceIndex, err) + return err + } + + // set MTU + for _, iface := range interfaces { + if uint32(mtu) != iface.NlMtu { + _mtu := uint32(mtu) + if err = util.SetNetIPInterface(iface.InterfaceIndex, &iface.AddressFamily, &_mtu, nil, nil); err != nil { + klog.Error(err) + return err + } + } + } + + return nil +} + +// Add host nic to external bridge +// Mac address, MTU, IP addresses & routes will be copied/transferred to the external bridge +func configProviderNic(nicName, brName string) (int, error) { + // TODO + return 0, nil +} + +// Remove host nic from external bridge +// IP addresses & routes will be transferred to the host nic +func removeProviderNic(nicName, brName string) error { + // TODO + return nil +} + +func turnOffNicTxChecksum(nicName string) error { + // TODO + return nil +} + +func getShortSharedDir(uid types.UID, volumeName string) string { + // DPDK is not supported on Windows + return "" +} diff --git a/pkg/ovs/ovs-vsctl.go b/pkg/ovs/ovs-vsctl.go index 5efa5c4de02..3c789a217be 100644 --- a/pkg/ovs/ovs-vsctl.go +++ b/pkg/ovs/ovs-vsctl.go @@ -195,73 +195,6 @@ func ClearPodBandwidth(podName, podNamespace, ifaceID string) error { return nil } -// SetInterfaceBandwidth set ingress/egress qos for given pod, annotation values are for node/pod -// but ingress/egress parameters here are from the point of ovs port/interface view, so reverse input parameters when call func SetInterfaceBandwidth -func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress, podPriority string) error { - ingressMPS, _ := strconv.Atoi(ingress) - ingressKPS := ingressMPS * 1000 - interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", iface)) - if err != nil { - return err - } - - qosIfaceUidMap, err := ListExternalIds("qos") - if err != nil { - return err - } - - queueIfaceUidMap, err := ListExternalIds("queue") - if err != nil { - return err - } - - for _, ifName := range interfaceList { - // ingress_policing_rate is in Kbps - err := ovsSet("interface", ifName, fmt.Sprintf("ingress_policing_rate=%d", ingressKPS), fmt.Sprintf("ingress_policing_burst=%d", ingressKPS*8/10)) - if err != nil { - return err - } - - egressMPS, _ := strconv.Atoi(egress) - egressBPS := egressMPS * 1000 * 1000 - - if egressBPS > 0 { - queueUid, err := SetHtbQosQueueRecord(podName, podNamespace, iface, podPriority, egressBPS, queueIfaceUidMap) - if err != nil { - return err - } - - if err = SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUid, qosIfaceUidMap); err != nil { - return err - } - } else { - if qosUid, ok := qosIfaceUidMap[iface]; ok { - qosType, err := ovsGet("qos", qosUid, "type", "") - if err != nil { - return err - } - if qosType != util.HtbQos { - continue - } - queueId, err := ovsGet("qos", qosUid, "queues", "0") - if err != nil { - return err - } - - // It's difficult to check if qos and queue should be destroyed here since can not get subnet info here. So leave destroy operation in loop check - if _, err := Exec("remove", "queue", queueId, "other_config", "max-rate"); err != nil { - return fmt.Errorf("failed to remove rate limit for queue in pod %v/%v, %v", podNamespace, podName, err) - } - } - } - - if err = SetHtbQosPriority(podName, podNamespace, iface, ifName, podPriority, qosIfaceUidMap, queueIfaceUidMap); err != nil { - return err - } - } - return nil -} - // CleanLostInterface will clean up related ovs port, interface and qos // When reboot node, the ovs internal interface will be deleted. func CleanLostInterface() { @@ -401,178 +334,6 @@ func GetResidualInternalPorts() []string { return residualPorts } -func ClearHtbQosQueue(podName, podNamespace, iface string) error { - var queueList []string - var err error - if iface != "" { - queueList, err = ovsFind("queue", "_uuid", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) - if err != nil { - return err - } - } else { - queueList, err = ovsFind("queue", "_uuid", fmt.Sprintf(`external-ids:pod="%s/%s"`, podNamespace, podName)) - if err != nil { - return err - } - } - - // https://github.com/kubeovn/kube-ovn/issues/1191 - qosQueueMap, err := ListQosQueueIds() - if err != nil { - return err - } - - for _, queueId := range queueList { - found := false - for _, usedQueueId := range qosQueueMap { - if queueId == usedQueueId { - found = true - break - } - } - if found { - continue - } - - if err := ovsDestroy("queue", queueId); err != nil { - return err - } - } - return nil -} - -func IsHtbQos(iface string) (bool, error) { - qosList, err := ovsFind("qos", "_uuid", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) - if err != nil { - return false, err - } - - for _, qos := range qosList { - qosType, err := ovsGet("qos", qos, "type", "") - if err != nil { - return false, err - } - if qosType == util.HtbQos { - return true, nil - } - } - return false, nil -} - -func SetHtbQosQueueRecord(podName, podNamespace, iface, priority string, maxRateBPS int, queueIfaceUidMap map[string]string) (string, error) { - var queueCommandValues []string - var err error - if maxRateBPS > 0 { - queueCommandValues = append(queueCommandValues, fmt.Sprintf("other_config:max-rate=%d", maxRateBPS)) - } - if priority != "" { - queueCommandValues = append(queueCommandValues, fmt.Sprintf("other_config:priority=%s", priority)) - } - - if queueUid, ok := queueIfaceUidMap[iface]; ok { - if err := ovsSet("queue", queueUid, queueCommandValues...); err != nil { - return queueUid, err - } - } else { - queueCommandValues = append(queueCommandValues, fmt.Sprintf("external-ids:iface-id=%s", iface)) - if podNamespace != "" && podName != "" { - queueCommandValues = append(queueCommandValues, fmt.Sprintf("external-ids:pod=%s/%s", podNamespace, podName)) - } - - var queueId string - if queueId, err = ovsCreate("queue", queueCommandValues...); err != nil { - return queueUid, err - } - queueIfaceUidMap[iface] = queueId - } - - return queueIfaceUidMap[iface], nil -} - -func SetHtbQosPriority(podName, podNamespace, iface, ifName, priority string, qosIfaceUidMap, queueIfaceUidMap map[string]string) error { - if priority != "" { - queueUid, err := SetHtbQosQueueRecord(podName, podNamespace, iface, priority, 0, queueIfaceUidMap) - if err != nil { - return err - } - - if err = SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUid, qosIfaceUidMap); err != nil { - return err - } - } else { - var qosUid string - var ok bool - if qosUid, ok = qosIfaceUidMap[iface]; !ok { - return nil - } - - qosType, err := ovsGet("qos", qosUid, "type", "") - if err != nil { - return err - } - if qosType != util.HtbQos { - return nil - } - queueId, err := ovsGet("qos", qosUid, "queues", "0") - if err != nil { - return err - } - - // It's difficult to check if qos and queue should be destroyed here since can not get subnet info here. So leave destroy operation in subnet loop check - if _, err := Exec("remove", "queue", queueId, "other_config", "priority"); err != nil { - return fmt.Errorf("failed to remove priority for queue in pod %v/%v, %v", podNamespace, podName, err) - } - } - - return nil -} - -// SetQosQueueBinding set qos related to queue record. -func SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUid string, qosIfaceUidMap map[string]string) error { - var qosCommandValues []string - qosCommandValues = append(qosCommandValues, fmt.Sprintf("queues:0=%s", queueUid)) - - if qosUid, ok := qosIfaceUidMap[iface]; !ok { - qosCommandValues = append(qosCommandValues, "type=linux-htb", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) - if podNamespace != "" && podName != "" { - qosCommandValues = append(qosCommandValues, fmt.Sprintf("external-ids:pod=%s/%s", podNamespace, podName)) - } - qos, err := ovsCreate("qos", qosCommandValues...) - if err != nil { - return err - } - err = ovsSet("port", ifName, fmt.Sprintf("qos=%s", qos)) - if err != nil { - return err - } - qosIfaceUidMap[iface] = qos - } else { - qosType, err := ovsGet("qos", qosUid, "type", "") - if err != nil { - return err - } - if qosType != util.HtbQos { - klog.Errorf("netem qos exists for pod %s/%s, conflict with current qos, will be changed to htb qos", podNamespace, podName) - qosCommandValues = append(qosCommandValues, "type=linux-htb") - } - - if qosType == util.HtbQos { - queueId, err := ovsGet("qos", qosUid, "queues", "0") - if err != nil { - return err - } - if queueId == queueUid { - return nil - } - } - - if err := ovsSet("qos", qosUid, qosCommandValues...); err != nil { - return err - } - } - return nil -} - // remove qos related to this port. func ClearPortQosBinding(ifaceID string) error { interfaceList, err := ovsFind("interface", "name", fmt.Sprintf(`external-ids:iface-id="%s"`, ifaceID)) @@ -588,119 +349,6 @@ func ClearPortQosBinding(ifaceID string) error { return nil } -// SetPodQosPriority set qos to this pod port. -func SetPodQosPriority(podName, podNamespace, ifaceID, priority string, qosIfaceUidMap, queueIfaceUidMap map[string]string) error { - interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", ifaceID)) - if err != nil { - return err - } - - for _, ifName := range interfaceList { - if err = SetHtbQosPriority(podName, podNamespace, ifaceID, ifName, priority, qosIfaceUidMap, queueIfaceUidMap); err != nil { - return err - } - } - return nil -} - -// The latency value expressed in us. -func SetNetemQos(podName, podNamespace, iface, latency, limit, loss string) error { - latencyMs, _ := strconv.Atoi(latency) - latencyUs := latencyMs * 1000 - limitPkts, _ := strconv.Atoi(limit) - lossPercent, _ := strconv.ParseFloat(loss, 64) - - interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", iface)) - if err != nil { - return err - } - - for _, ifName := range interfaceList { - qosList, err := GetQosList(podName, podNamespace, iface) - if err != nil { - return err - } - - var qosCommandValues []string - if latencyMs > 0 { - qosCommandValues = append(qosCommandValues, fmt.Sprintf("other_config:latency=%d", latencyUs)) - } - if limitPkts > 0 { - qosCommandValues = append(qosCommandValues, fmt.Sprintf("other_config:limit=%d", limitPkts)) - } - if lossPercent > 0 { - qosCommandValues = append(qosCommandValues, fmt.Sprintf("other_config:loss=%v", lossPercent)) - } - if latencyMs > 0 || limitPkts > 0 || lossPercent > 0 { - if len(qosList) == 0 { - qosCommandValues = append(qosCommandValues, "type=linux-netem", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) - if podNamespace != "" && podName != "" { - qosCommandValues = append(qosCommandValues, fmt.Sprintf("external-ids:pod=%s/%s", podNamespace, podName)) - } - - qos, err := ovsCreate("qos", qosCommandValues...) - if err != nil { - return err - } - err = ovsSet("port", ifName, fmt.Sprintf("qos=%s", qos)) - if err != nil { - return err - } - } else { - for _, qos := range qosList { - qosType, err := ovsGet("qos", qos, "type", "") - if err != nil { - return err - } - if qosType != util.NetemQos { - klog.Errorf("htb qos with higher priority exists for pod %v/%v, conflict with netem qos config, please delete htb qos first", podNamespace, podName) - return nil - } - - if err := ovsSet("qos", qos, qosCommandValues...); err != nil { - return err - } - - if latencyMs == 0 { - if err := ovsRemove("qos", qos, "other_config", "latency"); err != nil { - return err - } - } - if limitPkts == 0 { - if err := ovsRemove("qos", qos, "other_config", "limit"); err != nil { - return err - } - } - if lossPercent == 0 { - if err := ovsRemove("qos", qos, "other_config", "loss"); err != nil { - return err - } - } - } - } - } else { - for _, qos := range qosList { - qosType, _ := ovsGet("qos", qos, "type", "") - if qosType != util.NetemQos { - continue - } - - if err = ClearPortQosBinding(iface); err != nil { - klog.Errorf("failed to delete qos bingding info for interface %s: %v", iface, err) - return err - } - - // reuse this function to delete qos record - if err = ClearPodBandwidth(podName, podNamespace, iface); err != nil { - klog.Errorf("failed to delete netemqos record for pod %s/%s: %v", podNamespace, podName, err) - return err - } - } - } - } - return nil -} - func ListExternalIds(table string) (map[string]string, error) { args := []string{"--data=bare", "--format=csv", "--no-heading", "--columns=_uuid,external_ids", "find", table, "external_ids:iface-id!=[]"} output, err := Exec(args...) @@ -758,11 +406,3 @@ func ListQosQueueIds() (map[string]string, error) { } return result, nil } - -func IsUserspaceDataPath() (is bool, err error) { - dp, err := ovsFind("bridge", "datapath_type", "name=br-int") - if err != nil { - return false, err - } - return len(dp) > 0 && dp[0] == "netdev", nil -} diff --git a/pkg/ovs/ovs-vsctl_linux.go b/pkg/ovs/ovs-vsctl_linux.go new file mode 100644 index 00000000000..6a00407b010 --- /dev/null +++ b/pkg/ovs/ovs-vsctl_linux.go @@ -0,0 +1,373 @@ +package ovs + +import ( + "fmt" + "strconv" + + "k8s.io/klog/v2" + + "github.com/kubeovn/kube-ovn/pkg/util" +) + +// Glory belongs to openvswitch/ovn-kubernetes +// https://github.com/openvswitch/ovn-kubernetes/blob/master/go-controller/pkg/util/ovs.go + +// SetInterfaceBandwidth set ingress/egress qos for given pod, annotation values are for node/pod +// but ingress/egress parameters here are from the point of ovs port/interface view, so reverse input parameters when call func SetInterfaceBandwidth +func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress, podPriority string) error { + ingressMPS, _ := strconv.Atoi(ingress) + ingressKPS := ingressMPS * 1000 + interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", iface)) + if err != nil { + return err + } + + qosIfaceUidMap, err := ListExternalIds("qos") + if err != nil { + return err + } + + queueIfaceUidMap, err := ListExternalIds("queue") + if err != nil { + return err + } + + for _, ifName := range interfaceList { + // ingress_policing_rate is in Kbps + err := ovsSet("interface", ifName, fmt.Sprintf("ingress_policing_rate=%d", ingressKPS), fmt.Sprintf("ingress_policing_burst=%d", ingressKPS*8/10)) + if err != nil { + return err + } + + egressMPS, _ := strconv.Atoi(egress) + egressBPS := egressMPS * 1000 * 1000 + + if egressBPS > 0 { + queueUid, err := SetHtbQosQueueRecord(podName, podNamespace, iface, podPriority, egressBPS, queueIfaceUidMap) + if err != nil { + return err + } + + if err = SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUid, qosIfaceUidMap); err != nil { + return err + } + } else { + if qosUid, ok := qosIfaceUidMap[iface]; ok { + qosType, err := ovsGet("qos", qosUid, "type", "") + if err != nil { + return err + } + if qosType != util.HtbQos { + continue + } + queueId, err := ovsGet("qos", qosUid, "queues", "0") + if err != nil { + return err + } + + // It's difficult to check if qos and queue should be destroyed here since can not get subnet info here. So leave destroy operation in loop check + if _, err := Exec("remove", "queue", queueId, "other_config", "max-rate"); err != nil { + return fmt.Errorf("failed to remove rate limit for queue in pod %v/%v, %v", podNamespace, podName, err) + } + } + } + + if err = SetHtbQosPriority(podName, podNamespace, iface, ifName, podPriority, qosIfaceUidMap, queueIfaceUidMap); err != nil { + return err + } + } + return nil +} + +func ClearHtbQosQueue(podName, podNamespace, iface string) error { + var queueList []string + var err error + if iface != "" { + queueList, err = ovsFind("queue", "_uuid", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) + if err != nil { + return err + } + } else { + queueList, err = ovsFind("queue", "_uuid", fmt.Sprintf(`external-ids:pod="%s/%s"`, podNamespace, podName)) + if err != nil { + return err + } + } + + // https://github.com/kubeovn/kube-ovn/issues/1191 + qosQueueMap, err := ListQosQueueIds() + if err != nil { + return err + } + + for _, queueId := range queueList { + found := false + for _, usedQueueId := range qosQueueMap { + if queueId == usedQueueId { + found = true + break + } + } + if found { + continue + } + + if err := ovsDestroy("queue", queueId); err != nil { + return err + } + } + return nil +} + +func IsHtbQos(iface string) (bool, error) { + qosList, err := ovsFind("qos", "_uuid", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) + if err != nil { + return false, err + } + + for _, qos := range qosList { + qosType, err := ovsGet("qos", qos, "type", "") + if err != nil { + return false, err + } + if qosType == util.HtbQos { + return true, nil + } + } + return false, nil +} + +func SetHtbQosQueueRecord(podName, podNamespace, iface, priority string, maxRateBPS int, queueIfaceUidMap map[string]string) (string, error) { + var queueCommandValues []string + var err error + if maxRateBPS > 0 { + queueCommandValues = append(queueCommandValues, fmt.Sprintf("other_config:max-rate=%d", maxRateBPS)) + } + if priority != "" { + queueCommandValues = append(queueCommandValues, fmt.Sprintf("other_config:priority=%s", priority)) + } + + if queueUid, ok := queueIfaceUidMap[iface]; ok { + if err := ovsSet("queue", queueUid, queueCommandValues...); err != nil { + return queueUid, err + } + } else { + queueCommandValues = append(queueCommandValues, fmt.Sprintf("external-ids:iface-id=%s", iface)) + if podNamespace != "" && podName != "" { + queueCommandValues = append(queueCommandValues, fmt.Sprintf("external-ids:pod=%s/%s", podNamespace, podName)) + } + + var queueId string + if queueId, err = ovsCreate("queue", queueCommandValues...); err != nil { + return queueUid, err + } + queueIfaceUidMap[iface] = queueId + } + + return queueIfaceUidMap[iface], nil +} + +// SetPodQosPriority set qos to this pod port. +func SetPodQosPriority(podName, podNamespace, ifaceID, priority string, qosIfaceUidMap, queueIfaceUidMap map[string]string) error { + interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", ifaceID)) + if err != nil { + return err + } + + for _, ifName := range interfaceList { + if err = SetHtbQosPriority(podName, podNamespace, ifaceID, ifName, priority, qosIfaceUidMap, queueIfaceUidMap); err != nil { + return err + } + } + return nil +} + +func SetHtbQosPriority(podName, podNamespace, iface, ifName, priority string, qosIfaceUidMap, queueIfaceUidMap map[string]string) error { + if priority != "" { + queueUid, err := SetHtbQosQueueRecord(podName, podNamespace, iface, priority, 0, queueIfaceUidMap) + if err != nil { + return err + } + + if err = SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUid, qosIfaceUidMap); err != nil { + return err + } + } else { + var qosUid string + var ok bool + if qosUid, ok = qosIfaceUidMap[iface]; !ok { + return nil + } + + qosType, err := ovsGet("qos", qosUid, "type", "") + if err != nil { + return err + } + if qosType != util.HtbQos { + return nil + } + queueId, err := ovsGet("qos", qosUid, "queues", "0") + if err != nil { + return err + } + + // It's difficult to check if qos and queue should be destroyed here since can not get subnet info here. So leave destroy operation in subnet loop check + if _, err := Exec("remove", "queue", queueId, "other_config", "priority"); err != nil { + return fmt.Errorf("failed to remove priority for queue in pod %v/%v, %v", podNamespace, podName, err) + } + } + + return nil +} + +// SetQosQueueBinding set qos related to queue record. +func SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUid string, qosIfaceUidMap map[string]string) error { + var qosCommandValues []string + qosCommandValues = append(qosCommandValues, fmt.Sprintf("queues:0=%s", queueUid)) + + if qosUid, ok := qosIfaceUidMap[iface]; !ok { + qosCommandValues = append(qosCommandValues, "type=linux-htb", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) + if podNamespace != "" && podName != "" { + qosCommandValues = append(qosCommandValues, fmt.Sprintf("external-ids:pod=%s/%s", podNamespace, podName)) + } + qos, err := ovsCreate("qos", qosCommandValues...) + if err != nil { + return err + } + err = ovsSet("port", ifName, fmt.Sprintf("qos=%s", qos)) + if err != nil { + return err + } + qosIfaceUidMap[iface] = qos + } else { + qosType, err := ovsGet("qos", qosUid, "type", "") + if err != nil { + return err + } + if qosType != util.HtbQos { + klog.Errorf("netem qos exists for pod %s/%s, conflict with current qos, will be changed to htb qos", podNamespace, podName) + qosCommandValues = append(qosCommandValues, "type=linux-htb") + } + + if qosType == util.HtbQos { + queueId, err := ovsGet("qos", qosUid, "queues", "0") + if err != nil { + return err + } + if queueId == queueUid { + return nil + } + } + + if err := ovsSet("qos", qosUid, qosCommandValues...); err != nil { + return err + } + } + return nil +} + +// The latency value expressed in us. +func SetNetemQos(podName, podNamespace, iface, latency, limit, loss string) error { + latencyMs, _ := strconv.Atoi(latency) + latencyUs := latencyMs * 1000 + limitPkts, _ := strconv.Atoi(limit) + lossPercent, _ := strconv.ParseFloat(loss, 64) + + interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", iface)) + if err != nil { + return err + } + + for _, ifName := range interfaceList { + qosList, err := GetQosList(podName, podNamespace, iface) + if err != nil { + return err + } + + var qosCommandValues []string + if latencyMs > 0 { + qosCommandValues = append(qosCommandValues, fmt.Sprintf("other_config:latency=%d", latencyUs)) + } + if limitPkts > 0 { + qosCommandValues = append(qosCommandValues, fmt.Sprintf("other_config:limit=%d", limitPkts)) + } + if lossPercent > 0 { + qosCommandValues = append(qosCommandValues, fmt.Sprintf("other_config:loss=%v", lossPercent)) + } + if latencyMs > 0 || limitPkts > 0 || lossPercent > 0 { + if len(qosList) == 0 { + qosCommandValues = append(qosCommandValues, "type=linux-netem", fmt.Sprintf(`external-ids:iface-id="%s"`, iface)) + if podNamespace != "" && podName != "" { + qosCommandValues = append(qosCommandValues, fmt.Sprintf("external-ids:pod=%s/%s", podNamespace, podName)) + } + + qos, err := ovsCreate("qos", qosCommandValues...) + if err != nil { + return err + } + err = ovsSet("port", ifName, fmt.Sprintf("qos=%s", qos)) + if err != nil { + return err + } + } else { + for _, qos := range qosList { + qosType, err := ovsGet("qos", qos, "type", "") + if err != nil { + return err + } + if qosType != util.NetemQos { + klog.Errorf("htb qos with higher priority exists for pod %v/%v, conflict with netem qos config, please delete htb qos first", podNamespace, podName) + return nil + } + + if err := ovsSet("qos", qos, qosCommandValues...); err != nil { + return err + } + + if latencyMs == 0 { + if err := ovsRemove("qos", qos, "other_config", "latency"); err != nil { + return err + } + } + if limitPkts == 0 { + if err := ovsRemove("qos", qos, "other_config", "limit"); err != nil { + return err + } + } + if lossPercent == 0 { + if err := ovsRemove("qos", qos, "other_config", "loss"); err != nil { + return err + } + } + } + } + } else { + for _, qos := range qosList { + qosType, _ := ovsGet("qos", qos, "type", "") + if qosType != util.NetemQos { + continue + } + + if err = ClearPortQosBinding(iface); err != nil { + klog.Errorf("failed to delete qos bingding info for interface %s: %v", iface, err) + return err + } + + // reuse this function to delete qos record + if err = ClearPodBandwidth(podName, podNamespace, iface); err != nil { + klog.Errorf("failed to delete netemqos record for pod %s/%s: %v", podNamespace, podName, err) + return err + } + } + } + } + return nil +} + +func IsUserspaceDataPath() (is bool, err error) { + dp, err := ovsFind("bridge", "datapath_type", "name=br-int") + if err != nil { + return false, err + } + return len(dp) > 0 && dp[0] == "netdev", nil +} diff --git a/pkg/ovs/ovs-vsctl_windows.go b/pkg/ovs/ovs-vsctl_windows.go new file mode 100644 index 00000000000..59d6444b2fb --- /dev/null +++ b/pkg/ovs/ovs-vsctl_windows.go @@ -0,0 +1,49 @@ +package ovs + +// SetInterfaceBandwidth set ingress/egress qos for given pod, annotation values are for node/pod +// but ingress/egress parameters here are from the point of ovs port/interface view, so reverse input parameters when call func SetInterfaceBandwidth +func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress, podPriority string) error { + // TODO + return nil +} + +func ClearHtbQosQueue(podName, podNamespace, iface string) error { + //TODO + return nil +} + +func IsHtbQos(iface string) (bool, error) { + // TODO + return false, nil +} + +func SetHtbQosQueueRecord(podName, podNamespace, iface, priority string, maxRateBPS int, queueIfaceUidMap map[string]string) (string, error) { + //TODO + return "", nil +} + +// SetPodQosPriority set qos to this pod port. +func SetPodQosPriority(podName, podNamespace, ifaceID, priority string, qosIfaceUidMap, queueIfaceUidMap map[string]string) error { + //TODO + return nil +} + +// SetQosQueueBinding set qos related to queue record. +func SetQosQueueBinding(podName, podNamespace, ifName, iface, queueUid string, qosIfaceUidMap map[string]string) error { + // TODO + return nil +} + +// The latency value expressed in us. +func SetNetemQos(podName, podNamespace, iface, latency, limit, loss string) error { + // TODO + return nil +} + +func IsUserspaceDataPath() (is bool, err error) { + dp, err := ovsFind("bridge", "datapath_type", "name=br-int") + if err != nil { + return false, err + } + return len(dp) > 0 && dp[0] == "netdev", nil +} diff --git a/pkg/util/const_linux.go b/pkg/util/const_linux.go index a9052d0eba2..539c4fb8bd1 100644 --- a/pkg/util/const_linux.go +++ b/pkg/util/const_linux.go @@ -14,4 +14,7 @@ const ( KoDir = "/tmp/" KoENV = "MODULES" RpmENV = "RPMS" + + DefaultCniConfigFile = "/kube-ovn/01-kube-ovn.conflist" + DefaultCniConfigDir = "/etc/cni/net.d" ) diff --git a/pkg/util/const_windows.go b/pkg/util/const_windows.go new file mode 100644 index 00000000000..0e49747ba24 --- /dev/null +++ b/pkg/util/const_windows.go @@ -0,0 +1,11 @@ +package util + +const ( + WindowsListenPipe = `\\.\pipe\kube-ovn-daemon` + ChassisLoc = "system-id.conf" + + HnsNetwork = "kube-ovn" + + DefaultCniConfigFile = "01-kube-ovn.conflist" + DefaultCniConfigDir = `C:\etc\cni\net.d` +) diff --git a/pkg/util/link_windows.go b/pkg/util/link_windows.go new file mode 100644 index 00000000000..ed97a15dc3d --- /dev/null +++ b/pkg/util/link_windows.go @@ -0,0 +1,21 @@ +package util + +import "k8s.io/klog/v2" + +// SetLinkUp sets a link up +func SetLinkUp(name string) error { + adapter, err := GetNetAdapter(name, false) + if err != nil { + klog.Errorf("failed to get network adapter %s: %v", name, err) + return err + } + + if adapter.InterfaceAdminStatus != 1 { + if err = EnableAdapter(name); err != nil { + klog.Error(err) + return err + } + } + + return nil +} diff --git a/pkg/util/network_windows.go b/pkg/util/network_windows.go new file mode 100644 index 00000000000..a9ca321c955 --- /dev/null +++ b/pkg/util/network_windows.go @@ -0,0 +1,272 @@ +package util + +import ( + "encoding/json" + "fmt" + "strings" + + ps "github.com/bhendo/go-powershell" + "github.com/bhendo/go-powershell/backend" + "k8s.io/klog/v2" +) + +const ( + WindowsAddressFamilyV4 = 2 + WindowsAddressFamilyV6 = 23 +) + +// NetAdapter represents a network adapter on windows +type NetAdapter struct { + Name string + ElementName string + MacAddress string + InterfaceIndex uint32 + InterfaceAdminStatus uint32 + InterfaceOperationalStatus uint32 +} + +// NetIPInterface represents a net IP interface on windows +type NetIPInterface struct { + InterfaceIndex uint32 + InterfaceAlias string + AddressFamily uint16 + NlMtu uint32 + Forwarding uint8 + Dhcp uint8 +} + +// NetIPAddress represents a net IP address on windows +type NetIPAddress struct { + InterfaceIndex uint32 + InterfaceAlias string + AddressFamily uint16 + IPAddress string + PrefixLength uint8 +} + +// NetRoute represents a net route on windows +type NetRoute struct { + InterfaceIndex uint32 + InterfaceAlias string + AddressFamily uint16 + DestinationPrefix string + NextHop string +} + +func powershell(cmd string) (string, error) { + shell, err := ps.New(&backend.Local{}) + if err != nil { + return "", err + } + defer shell.Exit() + + stdout, _, err := shell.Execute(cmd) + if err != nil { + return stdout, err + } + return stdout, nil +} + +func bool2PsParam(v bool) string { + if v { + return "Enabled" + } + return "Disabled" +} + +func GetNetAdapter(name string, ignoreError bool) (*NetAdapter, error) { + output, err := powershell(fmt.Sprintf(`Get-NetAdapter -Name "%s" | ConvertTo-Json`, name)) + if err != nil { + if !ignoreError { + err2 := fmt.Errorf("failed to get network adapter %s: %v", name, err) + klog.Error(err2) + return nil, err2 + } + return nil, nil + } + + adapter := &NetAdapter{} + if err = json.Unmarshal([]byte(output), adapter); err != nil { + err2 := fmt.Errorf("failed to parse information of network adapter %s: %v", name, err) + klog.Error(err2) + return nil, err2 + } + + adapter.MacAddress = strings.ReplaceAll(adapter.MacAddress, "-", ":") + return adapter, nil +} + +func EnableAdapter(adapter string) error { + _, err := powershell(fmt.Sprintf(`Enable-NetAdapter -Name "%s" -Confirm:$False`, adapter)) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to enable network adapter %s: %v", adapter, err) + } + return nil +} + +func SetAdapterMac(adapter, mac string) error { + _, err := powershell(fmt.Sprintf(`Set-NetAdapter -Name "%s" -MacAddress %s -Confirm:$False`, adapter, mac)) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to set MAC address of network adapter %s: %v", adapter, err) + } + return nil +} + +func GetNetIPInterface(ifIndex uint32) ([]NetIPInterface, error) { + output, err := powershell(fmt.Sprintf("Get-NetIPInterface -InterfaceIndex %d | ConvertTo-Json", ifIndex)) + if err != nil { + err2 := fmt.Errorf("failed to get NetIPInterface with index %d: %v", ifIndex, err) + klog.Error(err2) + return nil, err2 + } + + result := make([]NetIPInterface, 0, 2) + if err = json.Unmarshal([]byte(output), &result); err != nil { + err2 := fmt.Errorf("failed to parse information of NetIPInterface: %v", err) + klog.Error(err2) + return nil, err2 + } + + return result, nil +} + +func SetNetIPInterface(ifIndex uint32, addressFamily *uint16, mtu *uint32, dhcp, forwarding *bool) error { + parameters := make([]string, 0) + if addressFamily != nil { + parameters = append(parameters, fmt.Sprintf("-AddressFamily %d", *addressFamily)) + } + if mtu != nil { + parameters = append(parameters, fmt.Sprintf("-NlMtuBytes %d", *mtu)) + } + if dhcp != nil { + parameters = append(parameters, fmt.Sprintf("-Dhcp %s", bool2PsParam(*dhcp))) + } + if forwarding != nil { + parameters = append(parameters, fmt.Sprintf("-Forwading %s", bool2PsParam(*forwarding))) + } + + _, err := powershell(fmt.Sprintf("Set-NetIPInterface -IncludeAllCompartments -InterfaceIndex %d %s -Confirm:$False", ifIndex, strings.Join(parameters, " "))) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to set NetIPInterface with index %d: %v", ifIndex, err) + } + + return nil +} + +func GetNetIPAddress(ifIndex uint32) ([]NetIPAddress, error) { + output, err := powershell(fmt.Sprintf("Get-NetIPAddress -InterfaceIndex %d | ConvertTo-Json", ifIndex)) + if err != nil { + err2 := fmt.Errorf("failed to get NetIPAddress with index %d: %v", ifIndex, err) + klog.Error(err2) + return nil, err2 + } + + if output[0] == '{' { + output = fmt.Sprintf("[%s]", output) + } + + result := make([]NetIPAddress, 0, 2) + if err = json.Unmarshal([]byte(output), &result); err != nil { + err2 := fmt.Errorf("failed to parse information of NetIPAddress: %v", err) + klog.Error(err2) + return nil, err2 + } + + return result, nil +} + +func NewNetIPAddress(ifIndex uint32, ipAddr string) error { + fields := strings.Split(ipAddr, "/") + cmd := fmt.Sprintf("New-NetIPAddress -InterfaceIndex %d -IPAddress %s -PrefixLength %s -PolicyStore ActiveStore -Confirm:$False", ifIndex, fields[0], fields[1]) + _, err := powershell(cmd) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to add IP address %s to interface with index %d: %v", ipAddr, ifIndex, err) + } + return nil +} + +func RemoveNetIPAddress(ifIndex uint32, ipAddr string) error { + fields := strings.Split(ipAddr, "/") + cmd := fmt.Sprintf("Remove-NetIPAddress -InterfaceIndex %d -IPAddress %s -PrefixLength %s -Confirm:$False", ifIndex, fields[0], fields[1]) + _, err := powershell(cmd) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to remove IP address %s from interface with index %d: %v", ipAddr, ifIndex, err) + } + return nil +} + +func GetInterfaceByIP(ip string) (*NetIPInterface, error) { + output, err := powershell(fmt.Sprintf(`Get-NetIPAddress | Where-Object -Property IPAddress -eq -Value "%s" | ConvertTo-Json`, ip)) + if err != nil { + err2 := fmt.Errorf("failed to get interface by IP %s: %v", ip, err) + klog.Error(err2) + return nil, err2 + } + if len(output) == 0 { + err = fmt.Errorf("interface with IP %s not found:") + klog.Error(err) + return nil, err + } + + var ipAddr NetIPAddress + if err = json.Unmarshal([]byte(output), &ipAddr); err != nil { + err2 := fmt.Errorf("failed to parse information of NetIPAddress: %v", err) + klog.Error(err2) + return nil, err2 + } + + interfaces, err := GetNetIPInterface(ipAddr.InterfaceIndex) + if err != nil { + return nil, err + } + for _, iface := range interfaces { + if iface.AddressFamily == ipAddr.AddressFamily { + return &iface, nil + } + } + + return nil, fmt.Errorf("failed to get interface with address family %d", ipAddr.AddressFamily, err) +} + +func GetNetRoute(ifIndex uint32) ([]NetRoute, error) { + output, err := powershell(fmt.Sprintf("Get-NetRoute -InterfaceIndex %d | ConvertTo-Json", ifIndex)) + if err != nil { + err2 := fmt.Errorf("failed to get NetRoute with index %d: %v", ifIndex, err) + klog.Error(err2) + return nil, err2 + } + + result := make([]NetRoute, 0, 2) + if err = json.Unmarshal([]byte(output), &result); err != nil { + err2 := fmt.Errorf("failed to parse information of NetRoute: %v", err) + klog.Error(err2) + return nil, err2 + } + + return result, nil +} + +func NewNetRoute(ifIndex uint32, destinationPrefix, nextHop string) error { + cmd := fmt.Sprintf("New-NetRoute -InterfaceIndex %d -DestinationPrefix %s -NextHop %s -PolicyStore ActiveStore -Confirm:$False", ifIndex, destinationPrefix, nextHop) + _, err := powershell(cmd) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to add route %s nexthop %s to interface with index %d: %v", destinationPrefix, nextHop, ifIndex, err) + } + return nil +} + +func RemoveNetRoute(ifIndex uint32, destinationPrefix string) error { + cmd := fmt.Sprintf("Remove-NetRoute -InterfaceIndex %d -DestinationPrefix %s -Confirm:$False", ifIndex, destinationPrefix) + _, err := powershell(cmd) + if err != nil { + klog.Error(err) + return fmt.Errorf("failed to remove route %s from interface with index %d: %v", destinationPrefix, ifIndex, err) + } + return nil +}