From 1a0749933ef8d7e6df75de1f2d152404f623b78f Mon Sep 17 00:00:00 2001 From: Berkay Tekin Oz Date: Fri, 7 Jun 2024 10:54:23 +0000 Subject: [PATCH] Initial version for k8sd-proxy --- .github/workflows/sync-images.yaml | 23 +++ .../controllers/ck8sconfig_controller.go | 33 ++- hack/sync-images.sh | 11 + hack/tools/go.mod | 19 ++ hack/tools/go.sum | 40 ++++ hack/tools/regsync.sh | 8 + hack/tools/tools.go | 5 + hack/upstream-images.yaml | 9 + pkg/ck8s/k8sd-proxy.yaml | 56 +++++ pkg/ck8s/k8sd_proxy.go | 192 ++++++++++++++++++ pkg/proxy/dial.go | 2 +- 11 files changed, 388 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/sync-images.yaml create mode 100755 hack/sync-images.sh create mode 100644 hack/tools/go.mod create mode 100644 hack/tools/go.sum create mode 100755 hack/tools/regsync.sh create mode 100644 hack/tools/tools.go create mode 100644 hack/upstream-images.yaml create mode 100644 pkg/ck8s/k8sd-proxy.yaml create mode 100644 pkg/ck8s/k8sd_proxy.go diff --git a/.github/workflows/sync-images.yaml b/.github/workflows/sync-images.yaml new file mode 100644 index 00000000..35a84bf8 --- /dev/null +++ b/.github/workflows/sync-images.yaml @@ -0,0 +1,23 @@ +name: Sync upstream images to ghcr.io + +on: + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "1.22" + + - name: Sync images + env: + USERNAME: ${{ github.actor }} + PASSWORD: ${{ secrets.GITHUB_TOKEN }} + run: | + ./hack/sync-images.sh \ No newline at end of file diff --git a/bootstrap/controllers/ck8sconfig_controller.go b/bootstrap/controllers/ck8sconfig_controller.go index a7ffcee8..600c6505 100644 --- a/bootstrap/controllers/ck8sconfig_controller.go +++ b/bootstrap/controllers/ck8sconfig_controller.go @@ -17,9 +17,11 @@ limitations under the License. package controllers import ( + "bytes" "context" "errors" "fmt" + "text/template" "time" "github.com/go-logr/logr" @@ -242,16 +244,29 @@ func (r *CK8sConfigReconciler) joinControlplane(ctx context.Context, scope *Scop return err } + var k8sdProxyDaemonset bytes.Buffer + + t, err := template.New("k8sd-proxy-daemonset").Parse(ck8s.K8sdProxyDaemonsetYaml) + if err != nil { + return err + } + + if err := t.Execute(&k8sdProxyDaemonset, struct { + K8sdPort int + }{ + K8sdPort: configStruct.ControlPlaneConfig.MicroclusterPort, + }); err != nil { + return err + } + // TODO(neoaggelos): figure out what is needed for k8sd proxy - // if scope.Config.Spec.IsEtcdEmbedded() { - // etcdProxyFile := bootstrapv1.File{ - // Path: etcd.EtcdProxyDaemonsetYamlLocation, - // Content: etcd.EtcdProxyDaemonsetYaml, - // Owner: "root:root", - // Permissions: "0640", - // } - // files = append(files, etcdProxyFile) - // } + k8sdProxyFile := bootstrapv1.File{ + Path: ck8s.K8sdProxyDaemonsetYamlLocation, + Content: k8sdProxyDaemonset.String(), + Owner: "root:root", + Permissions: "0400", + } + files = append(files, k8sdProxyFile) input := cloudinit.JoinControlPlaneInput{ BaseUserData: cloudinit.BaseUserData{ diff --git a/hack/sync-images.sh b/hack/sync-images.sh new file mode 100755 index 00000000..17e09c4c --- /dev/null +++ b/hack/sync-images.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Description: +# Sync images from upstream repositories under ghcr.io/canonical. +# +# Usage: +# $ USERNAME="$username" PASSWORD="$password" ./sync-images.sh + +DIR="$(realpath "$(dirname "${0}")")" + +"${DIR}/tools/regsync.sh" once -c "${DIR}/upstream-images.yaml" \ No newline at end of file diff --git a/hack/tools/go.mod b/hack/tools/go.mod new file mode 100644 index 00000000..321ce8c4 --- /dev/null +++ b/hack/tools/go.mod @@ -0,0 +1,19 @@ +module github.com/canonical/cluster-api-k8s/tools + +go 1.22.4 + +require github.com/regclient/regclient v0.6.1 + +require ( + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.17.8 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + golang.org/x/sys v0.20.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/hack/tools/go.sum b/hack/tools/go.sum new file mode 100644 index 00000000..42e110eb --- /dev/null +++ b/hack/tools/go.sum @@ -0,0 +1,40 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/olareg/olareg v0.1.0 h1:1dXBOgPrig5N7zoXyIZVQqU0QBo6sD9pbL6UYjY75CA= +github.com/olareg/olareg v0.1.0/go.mod h1:RBuU7JW7SoIIxZKzLRhq8sVtQeAHzCAtRrXEBx2KlM4= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/regclient/regclient v0.6.1 h1:4PxrGxMXrLpPrSaet8QZl568CVOolyHyukLL9UyogoU= +github.com/regclient/regclient v0.6.1/go.mod h1:hCKbRHYMx6LJntAhXzWVV7Oxyn9DzNVJoOKJaSnU5BM= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hack/tools/regsync.sh b/hack/tools/regsync.sh new file mode 100755 index 00000000..3c04a8c5 --- /dev/null +++ b/hack/tools/regsync.sh @@ -0,0 +1,8 @@ +#!/bin/bash -eu + +# Run regsync +TOOLS_DIR="$(realpath `dirname "${0}"`)" +( + cd "${TOOLS_DIR}" + go run github.com/regclient/regclient/cmd/regsync "${@}" +) \ No newline at end of file diff --git a/hack/tools/tools.go b/hack/tools/tools.go new file mode 100644 index 00000000..1c97d316 --- /dev/null +++ b/hack/tools/tools.go @@ -0,0 +1,5 @@ +package main + +import ( + _ "github.com/regclient/regclient/cmd/regsync" +) diff --git a/hack/upstream-images.yaml b/hack/upstream-images.yaml new file mode 100644 index 00000000..5d87a27d --- /dev/null +++ b/hack/upstream-images.yaml @@ -0,0 +1,9 @@ +version: 1 +creds: + - registry: ghcr.io + user: '{{ env "USERNAME" }}' + pass: '{{ env "PASSWORD" }}' +sync: + - source: alpine/socat:1.8.0.0 + target: ghcr.io/canonical/cluster-api-k8s/socat:1.8.0.0 + type: image \ No newline at end of file diff --git a/pkg/ck8s/k8sd-proxy.yaml b/pkg/ck8s/k8sd-proxy.yaml new file mode 100644 index 00000000..3ed22c45 --- /dev/null +++ b/pkg/ck8s/k8sd-proxy.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: k8sd-proxy-config + namespace: kube-system +data: + k8sd-port: "{{ .K8sdPort }}" +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: k8sd-proxy + namespace: kube-system + labels: + app: k8sd-proxy +spec: + selector: + matchLabels: + app: k8sd-proxy + template: + metadata: + labels: + app: k8sd-proxy + spec: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: k8sd-proxy + image: ghcr.io/canonical/cluster-api-k8s/socat:1.8.0.0 + env: + # TODO: Make this more robust by possibly finding/parsing the right IP. + # This works as a start but might not be sufficient as the kubelet IP might not match microcluster IP. + - name: HOSTIP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8SD_PORT + valueFrom: + configMapKeyRef: + name: k8sd-proxy-config + key: k8sd-port + args: + - TCP4-LISTEN:2380,fork,reuseaddr + - TCP4:$(HOSTIP):$(K8SD_PORT) + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 200Mi + terminationGracePeriodSeconds: 30 diff --git a/pkg/ck8s/k8sd_proxy.go b/pkg/ck8s/k8sd_proxy.go new file mode 100644 index 00000000..4b37a9b8 --- /dev/null +++ b/pkg/ck8s/k8sd_proxy.go @@ -0,0 +1,192 @@ +package ck8s + +import ( + "context" + "crypto/tls" + _ "embed" + "fmt" + "net/http" + "time" + + "github.com/canonical/cluster-api-k8s/pkg/proxy" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const K8sdProxyDaemonsetYamlLocation = "/opt/capi/manifests/k8sd-proxy.yaml" + +//go:embed k8sd-proxy.yaml +var K8sdProxyDaemonsetYaml string + +type K8sdProxy struct { + nodeIP string + client *http.Client +} + +type K8sdProxyGenerator struct { + restConfig *rest.Config + clientset *kubernetes.Clientset + proxyClientTimeout time.Duration + k8sdPort int +} + +func NewK8sdProxyGenerator(restConfig *rest.Config, clientset *kubernetes.Clientset, proxyClientTimeout time.Duration, k8sdPort int) (*K8sdProxyGenerator, error) { + return &K8sdProxyGenerator{ + restConfig: restConfig, + clientset: clientset, + proxyClientTimeout: proxyClientTimeout, + k8sdPort: k8sdPort, + }, nil +} + +func (g *K8sdProxyGenerator) forNodeName(ctx context.Context, nodeName string) (*K8sdProxy, error) { + node, err := g.clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "unable to get node in target cluster") + } + + return g.forNode(ctx, node) +} + +func (g *K8sdProxyGenerator) forNode(ctx context.Context, node *corev1.Node) (*K8sdProxy, error) { + podmap, err := g.getProxyPods(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get proxy pods: %w", err) + } + + podname, ok := podmap[node.Name] + if !ok { + return nil, fmt.Errorf("this node does not have a k8sd proxy pod") + } + + nodeInternalIP, err := g.getNodeInternalIP(node) + if err != nil { + return nil, fmt.Errorf("failed to get internal IP for node %s: %w", node.Name, err) + } + + client, err := g.NewHTTPClient(ctx, podname) + if err != nil { + return nil, err + } + + if err := g.checkIfK8sdIsReachable(ctx, client, nodeInternalIP); err != nil { + return nil, fmt.Errorf("failed to reach k8sd through proxy client: %w", err) + } + + return &K8sdProxy{ + nodeIP: nodeInternalIP, + client: client, + }, nil +} + +func (g *K8sdProxyGenerator) forControlPlane(ctx context.Context) (*K8sdProxy, error) { + cplaneNodes, err := g.getControlPlaneNodes(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get control plane nodes: %w", err) + } + + for _, node := range cplaneNodes.Items { + proxy, err := g.forNode(ctx, &node) + if err != nil { + continue + } + + return proxy, nil + } + + return nil, fmt.Errorf("failed to find a control plane node with a reachable k8sd proxy") +} + +func (g *K8sdProxyGenerator) checkIfK8sdIsReachable(ctx context.Context, client *http.Client, nodeIP string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s:%v/", nodeIP, g.k8sdPort), nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + res, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to reach k8sd through proxy client: %w", err) + } + res.Body.Close() + + return nil +} + +func (g *K8sdProxyGenerator) getNodeInternalIP(node *corev1.Node) (string, error) { + // TODO: Make this more robust by possibly finding/parsing the right IP. + // This works as a start but might not be sufficient as the kubelet IP might not match microcluster IP. + for _, addr := range node.Status.Addresses { + if addr.Type == "InternalIP" { + return addr.Address, nil + } + } + + return "", fmt.Errorf("unable to find internal IP for node %s", node.Name) +} + +func (g *K8sdProxyGenerator) getControlPlaneNodes(ctx context.Context) (*corev1.NodeList, error) { + nodes, err := g.clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/control-plane="}) + if err != nil { + return nil, errors.Wrap(err, "unable to list nodes in target cluster") + } + + if len(nodes.Items) == 0 { + return nil, errors.New("there isn't any nodes registered in target cluster") + } + + return nodes, nil +} + +func (g *K8sdProxyGenerator) getProxyPods(ctx context.Context) (map[string]string, error) { + pods, err := g.clientset.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{LabelSelector: "app=k8sd-proxy"}) + if err != nil { + return nil, errors.Wrap(err, "unable to list k8sd-proxy pods in target cluster") + } + + if len(pods.Items) == 0 { + return nil, errors.New("there isn't any k8sd-proxy pods in target cluster") + } + + podmap := make(map[string]string, len(pods.Items)) + for _, pod := range pods.Items { + podmap[pod.Spec.NodeName] = pod.Name + } + + return podmap, nil +} + +func (g *K8sdProxyGenerator) NewHTTPClient(ctx context.Context, podName string) (*http.Client, error) { + p := proxy.Proxy{ + Kind: "pods", + Namespace: metav1.NamespaceSystem, + ResourceName: podName, + KubeConfig: g.restConfig, + Port: 2380, + } + + dialer, err := proxy.NewDialer(p) + if err != nil { + return nil, fmt.Errorf("failed to create proxy dialer: %w", err) + } + + // We return a http client with the same parameters as http.DefaultClient + // and an overridden DialContext to proxy the requests through api server. + return &http.Client{ + Transport: &http.Transport{ + Proxy: http.DefaultTransport.(*http.Transport).Proxy, + DialContext: dialer.DialContext, + ForceAttemptHTTP2: http.DefaultTransport.(*http.Transport).ForceAttemptHTTP2, + MaxIdleConns: http.DefaultTransport.(*http.Transport).MaxIdleConns, + IdleConnTimeout: http.DefaultTransport.(*http.Transport).IdleConnTimeout, + TLSHandshakeTimeout: http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout, + ExpectContinueTimeout: http.DefaultTransport.(*http.Transport).ExpectContinueTimeout, + // TODO: Workaround for now, address later on + // get the certificate fingerprint from the matching node through a resource in the cluster (TBD), and validate it in the TLSClientConfig + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Timeout: g.proxyClientTimeout, + }, nil +} diff --git a/pkg/proxy/dial.go b/pkg/proxy/dial.go index e4c1024f..d3d80729 100644 --- a/pkg/proxy/dial.go +++ b/pkg/proxy/dial.go @@ -89,7 +89,7 @@ func (d *Dialer) DialContext(_ context.Context, _ string, addr string) (net.Conn Post(). Resource(d.proxy.Kind). Namespace(d.proxy.Namespace). - Name(addr). + Name(d.proxy.ResourceName). SubResource("portforward") dialer := spdy.NewDialer(d.upgrader, &http.Client{Transport: d.proxyTransport}, "POST", req.URL())