Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] update integration tests #882

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[WIP] update integration tests #882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 17 additions & 188 deletions controllers/dns_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,21 @@ package controllers
import (
"context"
"fmt"
"sort"
"strconv"
"strings"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
externaldns "sigs.k8s.io/external-dns/endpoint"
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"

kuadrantdnsv1alpha1 "github.com/kuadrant/dns-operator/api/v1alpha1"
"github.com/kuadrant/dns-operator/pkg/builder"

"github.com/kuadrant/kuadrant-operator/api/v1alpha1"
"github.com/kuadrant/kuadrant-operator/pkg/multicluster"
)

const (
LabelGatewayReference = "kuadrant.io/gateway"
LabelGatewayNSRef = "kuadrant.io/gateway-namespace"
LabelListenerReference = "kuadrant.io/listener-name"

DefaultTTL = 60
DefaultCnameTTL = 300
)

type dnsHelper struct {
Expand Down Expand Up @@ -56,182 +49,6 @@ func gatewayDNSRecordLabels(gwKey client.ObjectKey) map[string]string {
}
}

func (dh *dnsHelper) setEndpoints(mcgTarget *multicluster.GatewayTarget, dnsRecord *kuadrantdnsv1alpha1.DNSRecord, listener gatewayapiv1.Listener, loadBalancing *v1alpha1.LoadBalancingSpec) {
gwListenerHost := string(*listener.Hostname)
var endpoints []*externaldns.Endpoint

//Health Checks currently modify endpoints, so we have to keep existing ones in order to not lose health check ids
currentEndpoints := make(map[string]*externaldns.Endpoint, len(dnsRecord.Spec.Endpoints))
for _, endpoint := range dnsRecord.Spec.Endpoints {
currentEndpoints[getSetID(endpoint)] = endpoint
}

if loadBalancing == nil {
endpoints = dh.getSimpleEndpoints(mcgTarget, gwListenerHost, currentEndpoints)
} else {
endpoints = dh.getLoadBalancedEndpoints(mcgTarget, gwListenerHost, currentEndpoints)
}

sort.Slice(endpoints, func(i, j int) bool {
return getSetID(endpoints[i]) < getSetID(endpoints[j])
})

dnsRecord.Spec.Endpoints = endpoints
}

// getSimpleEndpoints returns the endpoints for the given GatewayTarget using the simple routing strategy

func (dh *dnsHelper) getSimpleEndpoints(mcgTarget *multicluster.GatewayTarget, hostname string, currentEndpoints map[string]*externaldns.Endpoint) []*externaldns.Endpoint {
var (
endpoints []*externaldns.Endpoint
ipValues []string
hostValues []string
)

for _, cgwTarget := range mcgTarget.ClusterGatewayTargets {
for _, gwa := range cgwTarget.Status.Addresses {
if *gwa.Type == gatewayapiv1.IPAddressType {
ipValues = append(ipValues, gwa.Value)
} else {
hostValues = append(hostValues, gwa.Value)
}
}
}

if len(ipValues) > 0 {
endpoint := createOrUpdateEndpoint(hostname, ipValues, kuadrantdnsv1alpha1.ARecordType, "", DefaultTTL, currentEndpoints)
endpoints = append(endpoints, endpoint)
}

//ToDO This could possibly result in an invalid record since you can't have multiple CNAME target values https://github.com/kuadrant/kuadrant-operator/issues/663
if len(hostValues) > 0 {
endpoint := createOrUpdateEndpoint(hostname, hostValues, kuadrantdnsv1alpha1.CNAMERecordType, "", DefaultTTL, currentEndpoints)
endpoints = append(endpoints, endpoint)
}

return endpoints
}

// getLoadBalancedEndpoints returns the endpoints for the given GatewayTarget using the loadbalanced routing strategy
//
// Builds an array of externaldns.Endpoint resources and sets them on the given DNSRecord. The endpoints expected are calculated
// from the GatewayTarget using the target Gateway (GatewayTarget.Gateway), the LoadBalancing Spec
// from the DNSPolicy attached to the target gateway (GatewayTarget.LoadBalancing) and the list of clusters the
// target gateway is currently placed on (GatewayTarget.ClusterGatewayTargets).
//
// GatewayTarget.ClusterGatewayTarget are grouped by Geo, in the case of Geo not being defined in the
// LoadBalancing Spec (Weighted only) an internal only Geo Code of "default" is used and all clusters added to it.
//
// A CNAME record is created for the target host (DNSRecord.name), pointing to a generated gateway lb host.
// A CNAME record for the gateway lb host is created for every Geo, with appropriate Geo information, pointing to a geo
// specific host.
// A CNAME record for the geo specific host is created for every Geo, with weight information for that target added,
// pointing to a target cluster hostname.
// An A record for the target cluster hostname is created for any IP targets retrieved for that cluster.
//
// Example(Weighted only)
//
// www.example.com CNAME lb-1ab1.www.example.com
// lb-1ab1.www.example.com CNAME geolocation * default.lb-1ab1.www.example.com
// default.lb-1ab1.www.example.com CNAME weighted 100 1bc1.lb-1ab1.www.example.com
// default.lb-1ab1.www.example.com CNAME weighted 100 aws.lb.com
// 1bc1.lb-1ab1.www.example.com A 192.22.2.1
//
// Example(Geo, default IE)
//
// shop.example.com CNAME lb-a1b2.shop.example.com
// lb-a1b2.shop.example.com CNAME geolocation ireland ie.lb-a1b2.shop.example.com
// lb-a1b2.shop.example.com geolocation australia aus.lb-a1b2.shop.example.com
// lb-a1b2.shop.example.com geolocation default ie.lb-a1b2.shop.example.com (set by the default geo option)
// ie.lb-a1b2.shop.example.com CNAME weighted 100 ab1.lb-a1b2.shop.example.com
// ie.lb-a1b2.shop.example.com CNAME weighted 100 aws.lb.com
// aus.lb-a1b2.shop.example.com CNAME weighted 100 ab2.lb-a1b2.shop.example.com
// aus.lb-a1b2.shop.example.com CNAME weighted 100 ab3.lb-a1b2.shop.example.com
// ab1.lb-a1b2.shop.example.com A 192.22.2.1 192.22.2.5
// ab2.lb-a1b2.shop.example.com A 192.22.2.3
// ab3.lb-a1b2.shop.example.com A 192.22.2.4

func (dh *dnsHelper) getLoadBalancedEndpoints(mcgTarget *multicluster.GatewayTarget, hostname string, currentEndpoints map[string]*externaldns.Endpoint) []*externaldns.Endpoint {
cnameHost := hostname
if isWildCardHost(hostname) {
cnameHost = strings.Replace(hostname, "*.", "", -1)
}

var endpoint *externaldns.Endpoint
endpoints := make([]*externaldns.Endpoint, 0)
lbName := strings.ToLower(fmt.Sprintf("klb.%s", cnameHost))

for geoCode, cgwTargets := range mcgTarget.GroupTargetsByGeo() {
geoLbName := strings.ToLower(fmt.Sprintf("%s.%s", geoCode, lbName))
var clusterEndpoints []*externaldns.Endpoint
for _, cgwTarget := range cgwTargets {
var ipValues []string
var hostValues []string
for _, gwa := range cgwTarget.Status.Addresses {
if *gwa.Type == gatewayapiv1.IPAddressType {
ipValues = append(ipValues, gwa.Value)
} else {
hostValues = append(hostValues, gwa.Value)
}
}

if len(ipValues) > 0 {
clusterLbName := strings.ToLower(fmt.Sprintf("%s-%s.%s", cgwTarget.GetShortCode(), mcgTarget.GetShortCode(), lbName))
endpoint = createOrUpdateEndpoint(clusterLbName, ipValues, kuadrantdnsv1alpha1.ARecordType, "", DefaultTTL, currentEndpoints)
clusterEndpoints = append(clusterEndpoints, endpoint)
hostValues = append(hostValues, clusterLbName)
}

for _, hostValue := range hostValues {
endpoint = createOrUpdateEndpoint(geoLbName, []string{hostValue}, kuadrantdnsv1alpha1.CNAMERecordType, hostValue, DefaultTTL, currentEndpoints)
endpoint.SetProviderSpecificProperty(kuadrantdnsv1alpha1.ProviderSpecificWeight, strconv.Itoa(cgwTarget.GetWeight()))
clusterEndpoints = append(clusterEndpoints, endpoint)
}
}
if len(clusterEndpoints) == 0 {
continue
}
endpoints = append(endpoints, clusterEndpoints...)

//Create lbName CNAME (lb-a1b2.shop.example.com -> <geoCode>.lb-a1b2.shop.example.com)
endpoint = createOrUpdateEndpoint(lbName, []string{geoLbName}, kuadrantdnsv1alpha1.CNAMERecordType, string(geoCode), DefaultCnameTTL, currentEndpoints)
endpoint.SetProviderSpecificProperty(kuadrantdnsv1alpha1.ProviderSpecificGeoCode, string(geoCode))
endpoints = append(endpoints, endpoint)

//Add a default geo (*) endpoint if the current target is the default geo
if mcgTarget.IsDefaultGeo() {
endpoint = createOrUpdateEndpoint(lbName, []string{geoLbName}, kuadrantdnsv1alpha1.CNAMERecordType, "default", DefaultCnameTTL, currentEndpoints)
endpoint.SetProviderSpecificProperty(kuadrantdnsv1alpha1.ProviderSpecificGeoCode, string(v1alpha1.WildcardGeo))
endpoints = append(endpoints, endpoint)
}
}

if len(endpoints) > 0 {
//Create gwListenerHost CNAME (shop.example.com -> lb-a1b2.shop.example.com)
endpoint = createOrUpdateEndpoint(hostname, []string{lbName}, kuadrantdnsv1alpha1.CNAMERecordType, "", DefaultCnameTTL, currentEndpoints)
endpoints = append(endpoints, endpoint)
}

return endpoints
}

func createOrUpdateEndpoint(dnsName string, targets externaldns.Targets, recordType kuadrantdnsv1alpha1.DNSRecordType, setIdentifier string,
recordTTL externaldns.TTL, currentEndpoints map[string]*externaldns.Endpoint) (endpoint *externaldns.Endpoint) {
ok := false
endpointID := dnsName + setIdentifier
if endpoint, ok = currentEndpoints[endpointID]; !ok {
endpoint = &externaldns.Endpoint{}
if setIdentifier != "" {
endpoint.SetIdentifier = setIdentifier
}
}
endpoint.DNSName = dnsName
endpoint.RecordType = string(recordType)
endpoint.Targets = targets
endpoint.RecordTTL = recordTTL
return endpoint
}

// removeDNSForDeletedListeners remove any DNSRecords that are associated with listeners that no longer exist in this gateway
func (dh *dnsHelper) removeDNSForDeletedListeners(ctx context.Context, upstreamGateway *gatewayapiv1.Gateway) error {
dnsList := &kuadrantdnsv1alpha1.DNSRecordList{}
Expand Down Expand Up @@ -277,10 +94,22 @@ func (dh *dnsHelper) deleteDNSRecordForListener(ctx context.Context, owner metav
return dh.Delete(ctx, &dnsRecord, &client.DeleteOptions{})
}

func isWildCardHost(host string) bool {
return strings.HasPrefix(host, "*")
// GatewayWrapper is a wrapper for gateway to implement interface form the builder
type GatewayWrapper struct {
*gatewayapiv1.Gateway
}

func NewGatewayWrapper(gateway *gatewayapiv1.Gateway) *GatewayWrapper {
return &GatewayWrapper{Gateway: gateway}
}

func getSetID(endpoint *externaldns.Endpoint) string {
return endpoint.DNSName + endpoint.SetIdentifier
func (g GatewayWrapper) GetAddresses() []builder.TargetAddress {
addresses := make([]builder.TargetAddress, len(g.Status.Addresses))
for i, address := range g.Status.Addresses {
addresses[i] = builder.TargetAddress{
Type: builder.AddressType(*address.Type),
Value: address.Value,
}
}
return addresses
}
64 changes: 33 additions & 31 deletions controllers/dnspolicy_dnsrecords.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,15 @@
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/client"
crlog "sigs.k8s.io/controller-runtime/pkg/log"
externaldns "sigs.k8s.io/external-dns/endpoint"
gatewayapiv1 "sigs.k8s.io/gateway-api/apis/v1"

kuadrantdnsv1alpha1 "github.com/kuadrant/dns-operator/api/v1alpha1"
"github.com/kuadrant/dns-operator/pkg/builder"

"github.com/kuadrant/kuadrant-operator/api/v1alpha1"
reconcilerutils "github.com/kuadrant/kuadrant-operator/pkg/library/reconcilers"
"github.com/kuadrant/kuadrant-operator/pkg/library/utils"
"github.com/kuadrant/kuadrant-operator/pkg/multicluster"
)

func (r *DNSPolicyReconciler) reconcileDNSRecords(ctx context.Context, dnsPolicy *v1alpha1.DNSPolicy, gwDiffObj *reconcilerutils.GatewayDiffs) error {
Expand All @@ -41,54 +42,43 @@
return nil
}

func (r *DNSPolicyReconciler) reconcileGatewayDNSRecords(ctx context.Context, gw *gatewayapiv1.Gateway, dnsPolicy *v1alpha1.DNSPolicy) error {
func (r *DNSPolicyReconciler) reconcileGatewayDNSRecords(ctx context.Context, gateway *gatewayapiv1.Gateway, dnsPolicy *v1alpha1.DNSPolicy) error {
log := crlog.FromContext(ctx)
clusterID, err := utils.GetClusterUID(ctx, r.Client())
if err != nil {
return fmt.Errorf("failed to generate cluster ID: %w", err)
}
gatewayWrapper := multicluster.NewGatewayWrapper(gw, clusterID)
if err := gatewayWrapper.Validate(); err != nil {
return err
}

if err := r.dnsHelper.removeDNSForDeletedListeners(ctx, gatewayWrapper.Gateway); err != nil {
if err = r.dnsHelper.removeDNSForDeletedListeners(ctx, gateway); err != nil {
log.V(3).Info("error removing DNS for deleted listeners")
return err
}

clusterGateways := gatewayWrapper.GetClusterGateways()

log.V(3).Info("checking gateway for attached routes ", "gateway", gatewayWrapper.Name, "clusterGateways", clusterGateways)
log.V(3).Info("checking gateway for attached routes ", "gateway", gateway.Name)

for _, listener := range gatewayWrapper.Spec.Listeners {
for _, listener := range gateway.Spec.Listeners {
listenerHost := *listener.Hostname
if listenerHost == "" {
log.Info("skipping listener no hostname assigned", listener.Name, "in ns ", gatewayWrapper.Namespace)
log.Info("skipping listener no hostname assigned", listener.Name, "in ns ", gateway.Namespace)

Check warning on line 62 in controllers/dnspolicy_dnsrecords.go

View check run for this annotation

Codecov / codecov/patch

controllers/dnspolicy_dnsrecords.go#L62 

Added line #L62 was not covered by tests
continue
}

listenerGateways := utils.Filter(clusterGateways, func(cgw multicluster.ClusterGateway) bool {
hasAttachedRoute := false
for _, statusListener := range cgw.Status.Listeners {
if string(statusListener.Name) == string(listener.Name) {
hasAttachedRoute = int(statusListener.AttachedRoutes) > 0
break
}
hasAttachedRoute := false
for _, statusListener := range gateway.Status.Listeners {
if string(listener.Name) == string(statusListener.Name) {
hasAttachedRoute = statusListener.AttachedRoutes > 0
}
return hasAttachedRoute
})
}

if len(listenerGateways) == 0 {
if !hasAttachedRoute {
// delete record
log.V(1).Info("no cluster gateways, deleting DNS record", " for listener ", listener.Name)
if err := r.dnsHelper.deleteDNSRecordForListener(ctx, gatewayWrapper, listener); client.IgnoreNotFound(err) != nil {
if err := r.dnsHelper.deleteDNSRecordForListener(ctx, gateway, listener); client.IgnoreNotFound(err) != nil {
return fmt.Errorf("failed to delete dns record for listener %s : %w", listener.Name, err)
}
continue
}

dnsRecord, err := r.desiredDNSRecord(gatewayWrapper, dnsPolicy, listener, listenerGateways)
dnsRecord, err := r.desiredDNSRecord(gateway, clusterID, dnsPolicy, listener)
if err != nil {
return err
}
Expand All @@ -107,7 +97,7 @@
return nil
}

func (r *DNSPolicyReconciler) desiredDNSRecord(gateway *multicluster.GatewayWrapper, dnsPolicy *v1alpha1.DNSPolicy, targetListener gatewayapiv1.Listener, clusterGateways []multicluster.ClusterGateway) (*kuadrantdnsv1alpha1.DNSRecord, error) {
func (r *DNSPolicyReconciler) desiredDNSRecord(gateway *gatewayapiv1.Gateway, clusterID string, dnsPolicy *v1alpha1.DNSPolicy, targetListener gatewayapiv1.Listener) (*kuadrantdnsv1alpha1.DNSRecord, error) {
rootHost := string(*targetListener.Hostname)
var healthCheckSpec *kuadrantdnsv1alpha1.HealthCheckSpec

Expand Down Expand Up @@ -136,13 +126,11 @@
}
dnsRecord.Labels[LabelListenerReference] = string(targetListener.Name)

mcgTarget, err := multicluster.NewGatewayTarget(gateway.Gateway, clusterGateways, dnsPolicy.Spec.LoadBalancing)
endpoints, err := buildEndpoints(clusterID, string(*targetListener.Hostname), gateway, dnsPolicy)
if err != nil {
return nil, fmt.Errorf("failed to create multi cluster gateway target for listener %s : %w", targetListener.Name, err)
return nil, fmt.Errorf("failed to generate dns record for a gateway %s in %s ns: %w", gateway.Name, gateway.Namespace, err)

Check warning on line 131 in controllers/dnspolicy_dnsrecords.go

View check run for this annotation

Codecov / codecov/patch

controllers/dnspolicy_dnsrecords.go#L131 

Added line #L131 was not covered by tests
}

r.dnsHelper.setEndpoints(mcgTarget, dnsRecord, targetListener, dnsPolicy.Spec.LoadBalancing)

dnsRecord.Spec.Endpoints = endpoints
return dnsRecord, nil
}

Expand Down Expand Up @@ -190,3 +178,17 @@

return true, nil
}

func buildEndpoints(clusterID, hostname string, gateway *gatewayapiv1.Gateway, policy *v1alpha1.DNSPolicy) ([]*externaldns.Endpoint, error) {
endpointBuilder := builder.NewEndpointsBuilder(NewGatewayWrapper(gateway), hostname)

if policy.Spec.LoadBalancing != nil {
endpointBuilder.WithLoadBalancingFor(
clusterID,
policy.Spec.LoadBalancing.Weight,
policy.Spec.LoadBalancing.Geo,
policy.Spec.LoadBalancing.DefaultGeo)
}

return endpointBuilder.Build()
}
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ require (
github.com/google/uuid v1.6.0
github.com/kuadrant/authorino v0.17.2
github.com/kuadrant/authorino-operator v0.11.1
github.com/kuadrant/dns-operator v0.0.0-20240809151102-e79ebbca8f70
github.com/kuadrant/dns-operator v0.0.0-20240926100317-2e2497411ab3
github.com/kuadrant/limitador-operator v0.9.0
github.com/kuadrant/policy-machinery v0.2.0
github.com/martinlindhe/base36 v1.1.1
Expand Down
Loading
Loading