Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add service outlier detection #7

Merged
merged 1 commit into from
Oct 19, 2023
Merged

add service outlier detection #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions apis/projectcontour/v1/detailedconditions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@ const (
// with an HTTPProxy resource which is not part of a delegation chain.
ConditionTypeOrphanedError = "Orphaned"

// ConditionTypeOutlierDetectionError describes an error condition with
// an HTTPProxy Outlier Detection issue.
ConditionTypeOutlierDetectionError = "OutlierDetectionError"

// ConditionTypePrefixReplaceError describes an error condition with
// an HTTPProxy path prefix replacement issue.
ConditionTypePrefixReplaceError = "PrefixReplaceError"
Expand Down
71 changes: 71 additions & 0 deletions apis/projectcontour/v1/httpproxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1036,6 +1036,77 @@ type Service struct {
// Slow start will gradually increase amount of traffic to a newly added endpoint.
// +optional
SlowStartPolicy *SlowStartPolicy `json:"slowStartPolicy,omitempty"`
// The policy for managing outlier detection on a service.
// If not specified, the global OutlierDetection policy will be used.
// +optional
OutlierDetection *OutlierDetection `json:"outlierDetection,omitempty"`
}

// OutlierDetection defines the configuration for outlier detection on a service.
type OutlierDetection struct {
// Disabled configures the Service to not use
// the default global OutlierDetection policy defined by the Contour configuration.
// Defaults to false.
// +optional
Disabled bool `json:"disabled,omitempty"`

// ConsecutiveServerErrors defines The number of consecutive server-side error responses before a consecutive 5xx ejection occurs.
// When the backend host encounters consecutive
// errors greater than or equal to ConsecutiveServerErrors, it will be
// ejected from the load balancing pool.
// for HTTP services, a 5xx counts as an error and for TCP services
// connection failures and connection timeouts count as an error.
// It can be disabled by setting the value to 0.
// Defaults to 5.
// +optional
ConsecutiveServerErrors *uint32 `json:"consecutiveServerErrors,omitempty"`

// Interval is the interval at which host status is evaluated.
// Defaults to 10s.
// +optional
// +kubebuilder:validation:Pattern=`^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$`
Interval *string `json:"interval,omitempty"`

// BaseEjectionTime is the base time that a host is ejected for.
// A host will remain ejected for a period of time equal to the
// product of the ejection base duration and the number of times the host has been ejected.
// Defaults to 30s.
// +optional
// +kubebuilder:validation:Pattern=`^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$`
BaseEjectionTime *string `json:"baseEjectionTime,omitempty"`

// MaxEjectionTime is the maximum time a host will be ejected for.
// After this amount of time, a host will be returned to normal operation.
// If not specified, the default value (300s) or BaseEjectionTime value is applied, whatever is larger.
// Defaults to 300s.
// +optional
// +kubebuilder:validation:Pattern=`^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$`
MaxEjectionTime *string `json:"maxEjectionTime,omitempty"`

// SplitExternalLocalOriginErrors defines whether to split the local origin errors from the external origin errors.
// Defaults to false.
// +optional
// +kubebuilder:default=false
SplitExternalLocalOriginErrors bool `json:"splitExternalLocalOriginErrors"`

// ConsecutiveLocalOriginFailure defines the number of consecutive local origin failures before a consecutive local origin ejection occurs.
// Parameters take effect only when SplitExternalLocalOriginErrors is true.
// Defaults to 5.
ConsecutiveLocalOriginFailure *uint32 `json:"consecutiveLocalOriginFailure,omitempty"`

// MaxEjectionPercent is the max percentage of hosts in the load balancing pool for the upstream service that can be ejected.
// But will eject at least one host regardless of the value here.
// Defaults to 10%.
// +optional
// +kubebuilder:validation:Maximum=100
MaxEjectionPercent *uint32 `json:"maxEjectionPercent,omitempty"`

// MaxEjectionTimeJitter is The maximum amount of jitter to add to the ejection time,
// in order to prevent a ‘thundering herd’ effect where all proxies try to reconnect to host at the same time.
// Defaults to 0s.
// +optional
// +kubebuilder:validation:Pattern=`^(((\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$`
MaxEjectionTimeJitter *string `json:"maxEjectionTimeJitter,omitempty"`
}

// HTTPHealthCheckPolicy defines health checks on the upstream service.
Expand Down
50 changes: 50 additions & 0 deletions apis/projectcontour/v1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions apis/projectcontour/v1alpha1/contourconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ type ContourConfigurationSpec struct {

// Tracing defines properties for exporting trace data to OpenTelemetry.
Tracing *TracingConfig `json:"tracing,omitempty"`

// GlobalOutlierDetection defines the configuration for outlier detection on all services.
// If defined, this will be used as the default for all services.
// +optional
GlobalOutlierDetection *contour_api_v1.OutlierDetection `json:"outlierDetection,omitempty"`
}

// XDSServerType is the type of xDS server implementation.
Expand Down
3 changes: 3 additions & 0 deletions changelogs/unreleased/5575-yangyy93-minor.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
## Add outlier detection related configuration detection for services

Add [outlier detection](https://www.envoyproxy.io/docs/envoy/v1.26.3/intro/arch_overview/upstream/outlier#arch-overview-outlier-detection) related configuration detection for services, including consecutiveServerErrors and localOriginal errors, and passive health checks can be performed on clusters.
3 changes: 3 additions & 0 deletions cmd/contour/serve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,7 @@ func (s *Server) doServe() error {
globalRateLimitService: contourConfiguration.RateLimitService,
maxRequestsPerConnection: contourConfiguration.Envoy.Cluster.MaxRequestsPerConnection,
perConnectionBufferLimitBytes: contourConfiguration.Envoy.Cluster.PerConnectionBufferLimitBytes,
globalOutlierDetection: contourConfiguration.GlobalOutlierDetection,
})

// Build the core Kubernetes event handler.
Expand Down Expand Up @@ -1085,6 +1086,7 @@ type dagBuilderConfig struct {
maxRequestsPerConnection *uint32
perConnectionBufferLimitBytes *uint32
globalRateLimitService *contour_api_v1alpha1.RateLimitServiceConfig
globalOutlierDetection *contour_api_v1.OutlierDetection
}

func (s *Server) getDAGBuilder(dbc dagBuilderConfig) *dag.Builder {
Expand Down Expand Up @@ -1177,6 +1179,7 @@ func (s *Server) getDAGBuilder(dbc dagBuilderConfig) *dag.Builder {
GlobalRateLimitService: dbc.globalRateLimitService,
PerConnectionBufferLimitBytes: dbc.perConnectionBufferLimitBytes,
SetSourceMetadataOnRoutes: true,
GlobalOutlierDetection: dbc.globalOutlierDetection,
},
}

Expand Down
126 changes: 126 additions & 0 deletions examples/contour/01-crds.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6280,6 +6280,69 @@ spec:
up corresponding endpoints which contain the ips to
route.
type: string
outlierDetection:
description: The policy for managing outlier detection
on a service.
properties:
baseEjectionTime:
description: BaseEjectionTime is the base time that
a host is ejected for. A host will remain ejected
for a period of time equal to the product of the
ejection base duration and the number of times the
host has been ejected. Defaults to 30s.
pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$
type: string
consecutiveLocalOriginFailure:
description: ConsecutiveLocalOriginFailure defines
the number of consecutive local origin failures
before a consecutive local origin ejection occurs.
Parameters take effect only when SplitExternalLocalOriginErrors
is true. Defaults to 5.
format: int32
type: integer
consecutiveServerErrors:
description: ConsecutiveServerErrors defines The number
of consecutive server-side error responses before
a consecutive 5xx ejection occurs. When the backend
host encounters consecutive errors greater than
or equal to ConsecutiveServerErrors, it will be
ejected from the load balancing pool. for HTTP services,
a 5xx counts as an error and for TCP services connection
failures and connection timeouts count as an error.
It can be disabled by setting the value to 0. Defaults
to 5.
format: int32
type: integer
interval:
description: Interval is the interval at which host
status is evaluated. Defaults to 10s.
pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$
type: string
maxEjectionPercent:
description: MaxEjectionPercent is the max percentage
of hosts in the load balancing pool for the upstream
service that can be ejected. But will eject at least
one host regardless of the value here. Defaults
to 10%.
format: int32
maximum: 100
type: integer
maxEjectionTime:
description: MaxEjectionTime is the maximum time a
host will be ejected for. After this amount of time,
a host will be returned to normal operation. If
not specified, the default value (300s) or BaseEjectionTime
value is applied, whatever is larger. Defaults to
300s.
pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$
type: string
splitExternalLocalOriginErrors:
default: false
description: SplitExternalLocalOriginErrors defines
whether to split the local origin errors from the
external origin errors. Defaults to false.
type: boolean
type: object
port:
description: Port (defined as Integer) to proxy traffic
to since a service can have multiple defined.
Expand Down Expand Up @@ -6677,6 +6740,69 @@ spec:
traffic. Names defined here will be used to look up corresponding
endpoints which contain the ips to route.
type: string
outlierDetection:
description: The policy for managing outlier detection on
a service.
properties:
baseEjectionTime:
description: BaseEjectionTime is the base time that
a host is ejected for. A host will remain ejected
for a period of time equal to the product of the ejection
base duration and the number of times the host has
been ejected. Defaults to 30s.
pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$
type: string
consecutiveLocalOriginFailure:
description: ConsecutiveLocalOriginFailure defines the
number of consecutive local origin failures before
a consecutive local origin ejection occurs. Parameters
take effect only when SplitExternalLocalOriginErrors
is true. Defaults to 5.
format: int32
type: integer
consecutiveServerErrors:
description: ConsecutiveServerErrors defines The number
of consecutive server-side error responses before
a consecutive 5xx ejection occurs. When the backend
host encounters consecutive errors greater than or
equal to ConsecutiveServerErrors, it will be ejected
from the load balancing pool. for HTTP services, a
5xx counts as an error and for TCP services connection
failures and connection timeouts count as an error.
It can be disabled by setting the value to 0. Defaults
to 5.
format: int32
type: integer
interval:
description: Interval is the interval at which host
status is evaluated. Defaults to 10s.
pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$
type: string
maxEjectionPercent:
description: MaxEjectionPercent is the max percentage
of hosts in the load balancing pool for the upstream
service that can be ejected. But will eject at least
one host regardless of the value here. Defaults to
10%.
format: int32
maximum: 100
type: integer
maxEjectionTime:
description: MaxEjectionTime is the maximum time a host
will be ejected for. After this amount of time, a
host will be returned to normal operation. If not
specified, the default value (300s) or BaseEjectionTime
value is applied, whatever is larger. Defaults to
300s.
pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms))+)$
type: string
splitExternalLocalOriginErrors:
default: false
description: SplitExternalLocalOriginErrors defines
whether to split the local origin errors from the
external origin errors. Defaults to false.
type: boolean
type: object
port:
description: Port (defined as Integer) to proxy traffic
to since a service can have multiple defined.
Expand Down
Loading
Loading