diff --git a/ingress/controllers/nginx/controller.go b/ingress/controllers/nginx/controller.go index 84653bab92..3f8776d7d0 100644 --- a/ingress/controllers/nginx/controller.go +++ b/ingress/controllers/nginx/controller.go @@ -42,6 +42,7 @@ import ( "k8s.io/contrib/ingress/controllers/nginx/healthcheck" "k8s.io/contrib/ingress/controllers/nginx/nginx" + "k8s.io/contrib/ingress/controllers/nginx/nginx/ratelimit" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" ) @@ -583,6 +584,12 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg nginx.NginxConfigur continue } + rl, err := ratelimit.ParseAnnotations(ing) + glog.V(3).Infof("nginx rate limit %v", rl) + if err != nil { + glog.V(3).Infof("error reading rate limit annotation in Ingress %v/%v: %v", ing.GetNamespace(), ing.GetName(), err) + } + host := rule.Host if host == "" { host = defServerName @@ -615,6 +622,7 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg nginx.NginxConfigur glog.V(3).Infof("error parsing rewrite annotations for Ingress rule %v/%v: %v", ing.GetNamespace(), ing.GetName(), err) } loc.Redirect = *locRew + loc.RateLimit = *rl addLoc = false continue @@ -635,9 +643,10 @@ func (lbc *loadBalancerController) getUpstreamServers(ngxCfg nginx.NginxConfigur } server.Locations = append(server.Locations, &nginx.Location{ - Path: nginxPath, - Upstream: *ups, - Redirect: *locRew, + Path: nginxPath, + Upstream: *ups, + Redirect: *locRew, + RateLimit: *rl, }) } } diff --git a/ingress/controllers/nginx/nginx.tmpl b/ingress/controllers/nginx/nginx.tmpl index 68277d3513..b12b0c9961 100644 --- a/ingress/controllers/nginx/nginx.tmpl +++ b/ingress/controllers/nginx/nginx.tmpl @@ -156,6 +156,12 @@ http { } {{ end }} + {{/* build all the required rate limit zones. Each annotation requires a dedicated zone */}} + {{/* 1MB -> 16 thousand 64-byte states or about 8 thousand 128-byte states */}} + {{ $zone := range (buildRateLimitZones .servers) }} + {{ $zone }} + {{ end }} + {{ range $server := .servers }} server { server_name {{ $server.Name }}; @@ -180,7 +186,10 @@ http { {{- range $location := $server.Locations }} {{- $path := buildLocation $location }} location {{ $path }} { - location {{ $path }} { + {{/* if the location contains a rate limit annotation, create one */}} + {{ $limits := buildRateLimit $location }} + {{- range $limit := $limits }} + {{ $limit }}{{ end }} proxy_set_header Host $host; # Pass Real IP diff --git a/ingress/controllers/nginx/nginx/nginx.go b/ingress/controllers/nginx/nginx/nginx.go index 7b2b58488c..3b1f26dd04 100644 --- a/ingress/controllers/nginx/nginx/nginx.go +++ b/ingress/controllers/nginx/nginx/nginx.go @@ -17,6 +17,7 @@ limitations under the License. package nginx import ( + "k8s.io/contrib/ingress/controllers/nginx/nginx/ratelimit" "k8s.io/contrib/ingress/controllers/nginx/nginx/rewrite" ) @@ -93,6 +94,7 @@ type Location struct { IsDefBackend bool Upstream Upstream Redirect rewrite.Redirect + RateLimit ratelimit.RateLimit } // LocationByPath sorts location by path diff --git a/ingress/controllers/nginx/nginx/ratelimit/main.go b/ingress/controllers/nginx/nginx/ratelimit/main.go new file mode 100644 index 0000000000..097d3b10f2 --- /dev/null +++ b/ingress/controllers/nginx/nginx/ratelimit/main.go @@ -0,0 +1,126 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ratelimit + +import ( + "errors" + "fmt" + "strconv" + + "k8s.io/kubernetes/pkg/apis/extensions" +) + +const ( + limitIp = "ingress.kubernetes.io/limit-connections" + limitRps = "ingress.kubernetes.io/limit-rps" + + // allow 5 times the specified limit as burst + defBurst = 5 + + // 1MB -> 16 thousand 64-byte states or about 8 thousand 128-byte states + // default is 5MB + defSharedSize = 5 +) + +var ( + // ErrInvalidRateLimit is returned when the annotation caontains invalid values + ErrInvalidRateLimit = errors.New("invalid rate limit value. Must be > 0") + + // ErrMissingAnnotations is returned when the ingress rule + // does not contains annotations related with rate limit + ErrMissingAnnotations = errors.New("no annotations present") +) + +// RateLimit returns rate limit configuration for an Ingress rule +// Is possible to limit the number of connections per IP address or +// connections per second. +// Note: Is possible to specify both limits +type RateLimit struct { + // Connections indicates a limit with the number of connections per IP address + Connections Zone + // RPS indicates a limit with the number of connections per second + RPS Zone +} + +// Zone returns information about the NGINX rate limit (limit_req_zone) +// http://nginx.org/en/docs/http/ngx_http_limit_req_module.html#limit_req_zone +type Zone struct { + Name string + Limit int + Burst int + // SharedSize amount of shared memory for the zone + SharedSize int +} + +type ingAnnotations map[string]string + +func (a ingAnnotations) limitIp() int { + val, ok := a[limitIp] + if ok { + if i, err := strconv.Atoi(val); err == nil { + return i + } + } + + return 0 +} + +func (a ingAnnotations) limitRps() int { + val, ok := a[limitRps] + if ok { + if i, err := strconv.Atoi(val); err == nil { + return i + } + } + + return 0 +} + +// ParseAnnotations parses the annotations contained in the ingress +// rule used to rewrite the defined paths +func ParseAnnotations(ing *extensions.Ingress) (*RateLimit, error) { + if ing.GetAnnotations() == nil { + return &RateLimit{}, ErrMissingAnnotations + } + + rps := ingAnnotations(ing.GetAnnotations()).limitRps() + conn := ingAnnotations(ing.GetAnnotations()).limitIp() + + if rps == 0 && conn == 0 { + return &RateLimit{ + Connections: Zone{}, + RPS: Zone{}, + }, ErrInvalidRateLimit + } + + zoneName := fmt.Sprintf("%v_%v", ing.GetNamespace(), ing.GetName()) + + return &RateLimit{ + Connections: Zone{ + Name: fmt.Sprintf("%v_conn", zoneName), + Limit: conn, + Burst: conn * defBurst, + SharedSize: defSharedSize, + }, + RPS: Zone{ + Name: fmt.Sprintf("%v_rps", zoneName), + Limit: rps, + Burst: conn * defBurst, + SharedSize: defSharedSize, + }, + }, nil +} diff --git a/ingress/controllers/nginx/nginx/ratelimit/main_test.go b/ingress/controllers/nginx/nginx/ratelimit/main_test.go new file mode 100644 index 0000000000..5242f57e97 --- /dev/null +++ b/ingress/controllers/nginx/nginx/ratelimit/main_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2015 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ratelimit + +import ( + "testing" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/apis/extensions" + "k8s.io/kubernetes/pkg/util/intstr" +) + +func buildIngress() *extensions.Ingress { + defaultBackend := extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &extensions.Ingress{ + ObjectMeta: api.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: extensions.IngressSpec{ + Backend: &extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []extensions.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestAnnotations(t *testing.T) { + ing := buildIngress() + + lip := ingAnnotations(ing.GetAnnotations()).limitIp() + if lip != 0 { + t.Error("Expected 0 in limit by ip but %v was returned", lip) + } + + lrps := ingAnnotations(ing.GetAnnotations()).limitRps() + if lrps != 0 { + t.Error("Expected 0 in limit by rps but %v was returend", lrps) + } + + data := map[string]string{} + data[limitIp] = "5" + data[limitRps] = "100" + ing.SetAnnotations(data) + + lip = ingAnnotations(ing.GetAnnotations()).limitIp() + if lip != 5 { + t.Error("Expected %v in limit by ip but %v was returend", lip) + } + + lrps = ingAnnotations(ing.GetAnnotations()).limitRps() + if lrps != 100 { + t.Error("Expected 100 in limit by rps but %v was returend", lrps) + } +} + +func TestWithoutAnnotations(t *testing.T) { + ing := buildIngress() + _, err := ParseAnnotations(ing) + if err == nil { + t.Error("Expected error with ingress without annotations") + } +} + +func TestBadRateLimiting(t *testing.T) { + ing := buildIngress() + + data := map[string]string{} + data[limitIp] = "0" + data[limitRps] = "0" + ing.SetAnnotations(data) + + _, err := ParseAnnotations(ing) + if err == nil { + t.Errorf("Expected error with invalid limits (0)") + } + + data = map[string]string{} + data[limitIp] = "5" + data[limitRps] = "100" + ing.SetAnnotations(data) + + rateLimit, err := ParseAnnotations(ing) + if err != nil { + t.Errorf("Uxpected error: %v", err) + } + + if rateLimit.Connections.Limit != 5 { + t.Error("Expected 5 in limit by ip but %v was returend", rateLimit.Connections) + } + + if rateLimit.RPS.Limit != 100 { + t.Error("Expected 100 in limit by rps but %v was returend", rateLimit.RPS) + } +} diff --git a/ingress/controllers/nginx/nginx/template.go b/ingress/controllers/nginx/nginx/template.go index a844cb944a..32e3a3350f 100644 --- a/ingress/controllers/nginx/nginx/template.go +++ b/ingress/controllers/nginx/nginx/template.go @@ -45,8 +45,10 @@ var ( return true }, - "buildLocation": buildLocation, - "buildProxyPass": buildProxyPass, + "buildLocation": buildLocation, + "buildProxyPass": buildProxyPass, + "buildRateLimitZones": buildRateLimitZones, + "buildRateLimit": buildRateLimit, } ) @@ -180,3 +182,59 @@ func buildProxyPass(input interface{}) string { // default proxy_pass return defProxyPass } + +// buildRateLimitZones produces an array of limit_conn_zone in order to allow +// rate limiting of request. Each Ingress rule could have up to two zones, one +// for connection limit by IP address and other for limiting request per second +func buildRateLimitZones(input interface{}) []string { + zones := []string{} + + servers, ok := input.([]*Server) + if !ok { + return zones + } + + for _, server := range servers { + for _, loc := range server.Locations { + + if loc.RateLimit.Connections.Limit != -1 { + zone := fmt.Sprintf("limit_conn_zone $binary_remote_addr zone=%v:%v;", + loc.RateLimit.Connections.Name, loc.RateLimit.Connections.SharedSize) + zones = append(zones, zone) + } + + if loc.RateLimit.RPS.Limit != -1 { + zone := fmt.Sprintf("limit_conn_zone $binary_remote_addr zone=%v:%v rate=%vr/s;", + loc.RateLimit.Connections.Name, loc.RateLimit.Connections.SharedSize, loc.RateLimit.Connections.Limit) + zones = append(zones, zone) + } + } + } + + return zones +} + +// buildRateLimit produces an array of limit_req to be used inside the Path of +// Ingress rules. The order: connections by IP first and RPS next. +func buildRateLimit(input interface{}) []string { + limits := []string{} + + loc, ok := input.(*Location) + if !ok { + return limits + } + + if loc.RateLimit.Connections.Limit != -1 { + limit := fmt.Sprintf("limit_conn %v %v;", + loc.RateLimit.Connections.Name, loc.RateLimit.Connections.Limit) + limits = append(limits, limit) + } + + if loc.RateLimit.RPS.Limit != -1 { + limit := fmt.Sprintf("limit_req zone=%v burst=%v nodelay;", + loc.RateLimit.Connections.Name, loc.RateLimit.Connections.Burst) + limits = append(limits, limit) + } + + return limits +}