From f27905b09a7e185e229b998978c64dd874103d10 Mon Sep 17 00:00:00 2001 From: areller Date: Wed, 20 May 2020 13:43:56 -0400 Subject: [PATCH] Adding health checks (#432) --- src/Microsoft.Tye.Core/ApplicationFactory.cs | 27 ++ .../ConfigModel/ConfigApplication.cs | 15 + .../ConfigModel/ConfigHttpProber.cs | 18 + .../ConfigModel/ConfigProbe.cs | 16 + .../ConfigModel/ConfigService.cs | 2 + .../ContainerServiceBuilder.cs | 4 + src/Microsoft.Tye.Core/CoreStrings.resx | 11 +- .../ExecutableServiceBuilder.cs | 4 + src/Microsoft.Tye.Core/HttpProberBuilder.cs | 16 + .../KubernetesManifestGenerator.cs | 68 +++ src/Microsoft.Tye.Core/ProbeBuilder.cs | 16 + .../ProjectServiceBuilder.cs | 4 + .../Serialization/ConfigServiceParser.cs | 173 +++++++ src/Microsoft.Tye.Hosting/DockerRunner.cs | 85 +++- src/Microsoft.Tye.Hosting/HttpProxyService.cs | 69 ++- src/Microsoft.Tye.Hosting/Model/HttpProber.cs | 17 + src/Microsoft.Tye.Hosting/Model/Probe.cs | 18 + .../Model/ReplicaBinding.cs | 13 + .../Model/ReplicaState.cs | 2 + .../Model/ReplicaStatus.cs | 8 + src/Microsoft.Tye.Hosting/Model/Service.cs | 5 + .../Model/ServiceDescription.cs | 2 + .../Model/V1/V1ReplicaStatus.cs | 1 + src/Microsoft.Tye.Hosting/PortAssigner.cs | 2 +- src/Microsoft.Tye.Hosting/ProcessRunner.cs | 9 +- src/Microsoft.Tye.Hosting/ProxyService.cs | 53 +- src/Microsoft.Tye.Hosting/ReplicaMonitor.cs | 455 ++++++++++++++++++ src/Microsoft.Tye.Hosting/TyeDashboardApi.cs | 3 +- src/Microsoft.Tye.Hosting/TyeHost.cs | 1 + src/tye/ApplicationBuilderExtensions.cs | 28 ++ test/E2ETest/HealthCheckTests.cs | 453 +++++++++++++++++ test/E2ETest/Microsoft.Tye.E2ETests.csproj | 3 +- test/E2ETest/ReplicaStoppingTests.cs | 143 ++++++ test/E2ETest/TyeGenerateTests.cs | 35 ++ .../testassets/generate/health-checks.yaml | 76 +++ .../projects/health-checks/api/Program.cs | 32 ++ .../api/Properties/launchSettings.json | 30 ++ .../projects/health-checks/api/Startup.cs | 166 +++++++ .../projects/health-checks/api/api.csproj | 8 + .../api/appsettings.Development.json | 9 + .../health-checks/api/appsettings.json | 10 + .../projects/health-checks/health-checks.sln | 34 ++ .../projects/health-checks/tye-all.yaml | 36 ++ .../projects/health-checks/tye-ingress.yaml | 30 ++ .../projects/health-checks/tye-liveness.yaml | 17 + .../projects/health-checks/tye-none.yaml | 10 + .../projects/health-checks/tye-proxy.yaml | 26 + .../projects/health-checks/tye-readiness.yaml | 17 + test/Test.Infrastructure/TestHelpers.cs | 127 +++-- 49 files changed, 2337 insertions(+), 70 deletions(-) create mode 100644 src/Microsoft.Tye.Core/ConfigModel/ConfigHttpProber.cs create mode 100644 src/Microsoft.Tye.Core/ConfigModel/ConfigProbe.cs create mode 100644 src/Microsoft.Tye.Core/HttpProberBuilder.cs create mode 100644 src/Microsoft.Tye.Core/ProbeBuilder.cs create mode 100644 src/Microsoft.Tye.Hosting/Model/HttpProber.cs create mode 100644 src/Microsoft.Tye.Hosting/Model/Probe.cs create mode 100644 src/Microsoft.Tye.Hosting/Model/ReplicaBinding.cs create mode 100644 src/Microsoft.Tye.Hosting/ReplicaMonitor.cs create mode 100644 test/E2ETest/HealthCheckTests.cs create mode 100644 test/E2ETest/ReplicaStoppingTests.cs create mode 100644 test/E2ETest/testassets/generate/health-checks.yaml create mode 100644 test/E2ETest/testassets/projects/health-checks/api/Program.cs create mode 100644 test/E2ETest/testassets/projects/health-checks/api/Properties/launchSettings.json create mode 100644 test/E2ETest/testassets/projects/health-checks/api/Startup.cs create mode 100644 test/E2ETest/testassets/projects/health-checks/api/api.csproj create mode 100644 test/E2ETest/testassets/projects/health-checks/api/appsettings.Development.json create mode 100644 test/E2ETest/testassets/projects/health-checks/api/appsettings.json create mode 100644 test/E2ETest/testassets/projects/health-checks/health-checks.sln create mode 100644 test/E2ETest/testassets/projects/health-checks/tye-all.yaml create mode 100644 test/E2ETest/testassets/projects/health-checks/tye-ingress.yaml create mode 100644 test/E2ETest/testassets/projects/health-checks/tye-liveness.yaml create mode 100644 test/E2ETest/testassets/projects/health-checks/tye-none.yaml create mode 100644 test/E2ETest/testassets/projects/health-checks/tye-proxy.yaml create mode 100644 test/E2ETest/testassets/projects/health-checks/tye-readiness.yaml diff --git a/src/Microsoft.Tye.Core/ApplicationFactory.cs b/src/Microsoft.Tye.Core/ApplicationFactory.cs index 522e1d732..54e15f729 100644 --- a/src/Microsoft.Tye.Core/ApplicationFactory.cs +++ b/src/Microsoft.Tye.Core/ApplicationFactory.cs @@ -97,6 +97,9 @@ public static async Task CreateAsync(OutputContext output, F } project.Replicas = configService.Replicas ?? 1; + project.Liveness = configService.Liveness != null ? GetProbeBuilder(configService.Liveness) : null; + project.Readiness = configService.Readiness != null ? GetProbeBuilder(configService.Readiness) : null; + // We don't apply more container defaults here because we might need // to prompt for the registry name. project.ContainerInfo = new ContainerInfo() { UseMultiphaseDockerfile = false, }; @@ -117,6 +120,9 @@ public static async Task CreateAsync(OutputContext output, F DockerFileContext = GetDockerFileContext(source, configService) }; service = container; + + container.Liveness = configService.Liveness != null ? GetProbeBuilder(configService.Liveness) : null; + container.Readiness = configService.Readiness != null ? GetProbeBuilder(configService.Readiness) : null; } else if (!string.IsNullOrEmpty(configService.Executable)) { @@ -139,6 +145,9 @@ public static async Task CreateAsync(OutputContext output, F Replicas = configService.Replicas ?? 1 }; service = executable; + + executable.Liveness = configService.Liveness != null ? GetProbeBuilder(configService.Liveness) : null; + executable.Readiness = configService.Readiness != null ? GetProbeBuilder(configService.Readiness) : null; } else if (!string.IsNullOrEmpty(configService.Include)) { @@ -379,5 +388,23 @@ private static void AddToRootServices(ApplicationBuilder root, HashSet d } } } + + private static ProbeBuilder GetProbeBuilder(ConfigProbe config) => new ProbeBuilder() + { + Http = config.Http != null ? GetHttpProberBuilder(config.Http) : null, + InitialDelay = config.InitialDelay, + Period = config.Period, + Timeout = config.Timeout, + SuccessThreshold = config.SuccessThreshold, + FailureThreshold = config.FailureThreshold + }; + + private static HttpProberBuilder GetHttpProberBuilder(ConfigHttpProber config) => new HttpProberBuilder() + { + Path = config.Path, + Headers = config.Headers, + Port = config.Port, + Protocol = config.Protocol + }; } } diff --git a/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs b/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs index 70d4e5d49..e9c262374 100644 --- a/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs +++ b/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs @@ -135,6 +135,21 @@ public void Validate() string.Join(Environment.NewLine, results.Select(r => r.ErrorMessage))); } } + + var probes = new[] { (Name: "liveness", Probe: service.Liveness), (Name: "readiness", Probe: service.Readiness) }.Where(p => p.Probe != null).ToArray(); + foreach (var probe in probes) + { + if (probe.Name == "liveness" && probe.Probe.SuccessThreshold != 1) + { + throw new TyeYamlException(CoreStrings.FormatSuccessThresholdMustBeOne(probe.Name)); + } + + // right now only http is supported, so it must be set + if (probe.Probe!.Http == null) + { + throw new TyeYamlException(CoreStrings.FormatProberRequired(probe.Name)); + } + } } foreach (var ingress in config.Ingress) diff --git a/src/Microsoft.Tye.Core/ConfigModel/ConfigHttpProber.cs b/src/Microsoft.Tye.Core/ConfigModel/ConfigHttpProber.cs new file mode 100644 index 000000000..905bcd8ed --- /dev/null +++ b/src/Microsoft.Tye.Core/ConfigModel/ConfigHttpProber.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Net.Http.Headers; + +namespace Microsoft.Tye.ConfigModel +{ + public class ConfigHttpProber + { + [Required] public string Path { get; set; } = default!; + public int? Port { get; set; } + public string? Protocol { get; set; } + public List> Headers { get; set; } = new List>(); + } +} diff --git a/src/Microsoft.Tye.Core/ConfigModel/ConfigProbe.cs b/src/Microsoft.Tye.Core/ConfigModel/ConfigProbe.cs new file mode 100644 index 000000000..28de62aaa --- /dev/null +++ b/src/Microsoft.Tye.Core/ConfigModel/ConfigProbe.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Tye.ConfigModel +{ + public class ConfigProbe + { + public ConfigHttpProber? Http { get; set; } + public int InitialDelay { get; set; } = 0; + public int Period { get; set; } = 1; + public int Timeout { get; set; } = 1; + public int SuccessThreshold { get; set; } = 1; + public int FailureThreshold { get; set; } = 3; + } +} diff --git a/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs b/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs index c9972f40e..86aef3d61 100644 --- a/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs +++ b/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs @@ -37,5 +37,7 @@ public class ConfigService [YamlMember(Alias = "env")] public List Configuration { get; set; } = new List(); public List BuildProperties { get; set; } = new List(); + public ConfigProbe? Liveness { get; set; } + public ConfigProbe? Readiness { get; set; } } } diff --git a/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs b/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs index df6b19b51..f9a0fee69 100644 --- a/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs +++ b/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs @@ -29,5 +29,9 @@ public ContainerServiceBuilder(string name, string image) public List EnvironmentVariables { get; } = new List(); public List Volumes { get; } = new List(); + + public ProbeBuilder? Liveness { get; set; } + + public ProbeBuilder? Readiness { get; set; } } } diff --git a/src/Microsoft.Tye.Core/CoreStrings.resx b/src/Microsoft.Tye.Core/CoreStrings.resx index 0904ae0cf..aaff68ec4 100644 --- a/src/Microsoft.Tye.Core/CoreStrings.resx +++ b/src/Microsoft.Tye.Core/CoreStrings.resx @@ -141,6 +141,12 @@ Cannot have multiple {0} bindings with the same port. + + A prober must be configured for the {0} probe. + + + "successThreshold" for {0} probe must be set to "1". + "{value}" must be a boolean value (true/false). @@ -150,6 +156,9 @@ "{value}" value cannot be negative. + + "{value}" value must be greater than zero. + Cannot have both "{0}" and "{1}" set for a service. Only one of project, image, and executable can be set for a given service. @@ -162,4 +171,4 @@ Unexpected key "{key}" in tye.yaml. - \ No newline at end of file + diff --git a/src/Microsoft.Tye.Core/ExecutableServiceBuilder.cs b/src/Microsoft.Tye.Core/ExecutableServiceBuilder.cs index 06e7f98d3..f0bfa6f89 100644 --- a/src/Microsoft.Tye.Core/ExecutableServiceBuilder.cs +++ b/src/Microsoft.Tye.Core/ExecutableServiceBuilder.cs @@ -23,5 +23,9 @@ public ExecutableServiceBuilder(string name, string executable) public int Replicas { get; set; } = 1; public List EnvironmentVariables { get; } = new List(); + + public ProbeBuilder? Liveness { get; set; } + + public ProbeBuilder? Readiness { get; set; } } } diff --git a/src/Microsoft.Tye.Core/HttpProberBuilder.cs b/src/Microsoft.Tye.Core/HttpProberBuilder.cs new file mode 100644 index 000000000..1abd7cd50 --- /dev/null +++ b/src/Microsoft.Tye.Core/HttpProberBuilder.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace Microsoft.Tye +{ + public class HttpProberBuilder + { + public string Path { get; set; } = default!; + public int? Port { get; set; } + public string? Protocol { get; set; } + public List> Headers { get; set; } = new List>(); + } +} diff --git a/src/Microsoft.Tye.Core/KubernetesManifestGenerator.cs b/src/Microsoft.Tye.Core/KubernetesManifestGenerator.cs index 1776643cf..33e27e1ce 100644 --- a/src/Microsoft.Tye.Core/KubernetesManifestGenerator.cs +++ b/src/Microsoft.Tye.Core/KubernetesManifestGenerator.cs @@ -378,6 +378,16 @@ public static KubernetesDeploymentOutput CreateDeployment( } } } + + if (project.Liveness != null) + { + AddProbe(project, container, project.Liveness!, "livenessProbe"); + } + + if (project.Readiness != null) + { + AddProbe(project, container, project.Readiness!, "readinessProbe"); + } } foreach (var sidecar in project.Sidecars) @@ -452,6 +462,64 @@ public static KubernetesDeploymentOutput CreateDeployment( return new KubernetesDeploymentOutput(project.Name, new YamlDocument(root)); } + private static void AddProbe(ServiceBuilder service, YamlMappingNode container, ProbeBuilder builder, string name) + { + var probe = new YamlMappingNode(); + container.Add(name, probe); + + if (builder.Http != null) + { + var builderHttp = builder.Http; + var httpGet = new YamlMappingNode(); + + probe.Add("httpGet", httpGet); + httpGet.Add("path", builderHttp.Path); + + if (builderHttp.Protocol != null) + { + httpGet.Add("scheme", builderHttp.Protocol.ToUpper()); + } + + if (builderHttp.Port != null) + { + httpGet.Add("port", builderHttp.Port.ToString()!); + } + else + { + // If port is not given, we pull default port + var binding = service.Bindings.First(b => builderHttp.Protocol == null || b.Protocol == builderHttp.Protocol); + if (binding.Port != null) + { + httpGet.Add("port", binding.Port.Value.ToString()); + } + + if (builderHttp.Protocol == null && binding.Protocol != null) + { + httpGet.Add("scheme", binding.Protocol.ToUpper()); + } + } + + if (builderHttp.Headers.Count > 0) + { + var headers = new YamlSequenceNode(); + httpGet.Add("httpHeaders", headers); + + foreach (var builderHeader in builderHttp.Headers) + { + var header = new YamlMappingNode(); + header.Add("name", builderHeader.Key); + header.Add("value", builderHeader.Value.ToString()!); + headers.Add(header); + } + } + } + + probe.Add("initialDelaySeconds", builder.InitialDelay.ToString()); + probe.Add("periodSeconds", builder.Period.ToString()!); + probe.Add("successThreshold", builder.SuccessThreshold.ToString()!); + probe.Add("failureThreshold", builder.FailureThreshold.ToString()!); + } + private static void AddEnvironmentVariablesForComputedBindings(YamlSequenceNode env, ComputedBindings bindings) { foreach (var binding in bindings.Bindings.OfType()) diff --git a/src/Microsoft.Tye.Core/ProbeBuilder.cs b/src/Microsoft.Tye.Core/ProbeBuilder.cs new file mode 100644 index 000000000..8c57dd002 --- /dev/null +++ b/src/Microsoft.Tye.Core/ProbeBuilder.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Tye +{ + public class ProbeBuilder + { + public HttpProberBuilder? Http { get; set; } + public int InitialDelay { get; set; } + public int Period { get; set; } + public int Timeout { get; set; } + public int SuccessThreshold { get; set; } + public int FailureThreshold { get; set; } + } +} diff --git a/src/Microsoft.Tye.Core/ProjectServiceBuilder.cs b/src/Microsoft.Tye.Core/ProjectServiceBuilder.cs index 46484751d..c4a551442 100644 --- a/src/Microsoft.Tye.Core/ProjectServiceBuilder.cs +++ b/src/Microsoft.Tye.Core/ProjectServiceBuilder.cs @@ -54,5 +54,9 @@ public ProjectServiceBuilder(string name, FileInfo projectFile) public Dictionary BuildProperties { get; } = new Dictionary(); public List Sidecars { get; } = new List(); + + public ProbeBuilder? Liveness { get; set; } + + public ProbeBuilder? Readiness { get; set; } } } diff --git a/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs b/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs index f9c845343..52b04ef88 100644 --- a/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs +++ b/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs @@ -3,6 +3,8 @@ // See the LICENSE file in the project root for more information. using System.Collections.Generic; +using System.Linq; +using Microsoft.Build.Evaluation; using Microsoft.Tye.ConfigModel; using YamlDotNet.RepresentationModel; @@ -37,6 +39,7 @@ private static void HandleServiceNameMapping(YamlMappingNode yamlMappingNode, Co { throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeABoolean(key)); } + service.External = external; break; case "image": @@ -56,6 +59,7 @@ private static void HandleServiceNameMapping(YamlMappingNode yamlMappingNode, Co { throw new TyeYamlException(child.Value.Start, CoreStrings.FormatExpectedYamlSequence(key)); } + HandleBuildProperties((child.Value as YamlSequenceNode)!, service.BuildProperties); break; case "include": @@ -69,6 +73,7 @@ private static void HandleServiceNameMapping(YamlMappingNode yamlMappingNode, Co { throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeABoolean(key)); } + service.Build = build; break; case "executable": @@ -115,8 +120,17 @@ private static void HandleServiceNameMapping(YamlMappingNode yamlMappingNode, Co { throw new TyeYamlException(child.Value.Start, CoreStrings.FormatExpectedYamlSequence(key)); } + HandleServiceConfiguration((child.Value as YamlSequenceNode)!, service.Configuration); break; + case "liveness": + service.Liveness = new ConfigProbe(); + HandleServiceProbe((YamlMappingNode)child.Value, service.Liveness!); + break; + case "readiness": + service.Readiness = new ConfigProbe(); + HandleServiceProbe((YamlMappingNode)child.Value, service.Readiness!); + break; default: throw new TyeYamlException(child.Key.Start, CoreStrings.FormatUnrecognizedKey(key)); } @@ -187,6 +201,165 @@ private static void HandleServiceVolumes(YamlSequenceNode yamlSequenceNode, List } } + private static void HandleServiceProbe(YamlMappingNode yamlMappingNode, ConfigProbe probe) + { + foreach (var child in yamlMappingNode.Children) + { + var key = YamlParser.GetScalarValue(child.Key); + + switch (key) + { + case "http": + probe.Http = new ConfigHttpProber(); + HandleServiceHttpProber((YamlMappingNode)child.Value, probe.Http!); + break; + case "initialDelay": + if (!int.TryParse(YamlParser.GetScalarValue(key, child.Value), out var initialDelay)) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeAnInteger(key)); + } + + if (initialDelay < 0) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBePositive(key)); + } + + probe.InitialDelay = initialDelay; + break; + case "period": + if (!int.TryParse(YamlParser.GetScalarValue(key, child.Value), out var period)) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeAnInteger(key)); + } + + if (period < 1) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeGreaterThanZero(key)); + } + + probe.Period = period; + break; + case "timeout": + if (!int.TryParse(YamlParser.GetScalarValue(key, child.Value), out var timeout)) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeAnInteger(key)); + } + + if (timeout < 1) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeGreaterThanZero(key)); + } + + probe.Timeout = timeout; + break; + case "successThreshold": + if (!int.TryParse(YamlParser.GetScalarValue(key, child.Value), out var successThreshold)) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeAnInteger(key)); + } + + if (successThreshold < 1) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeGreaterThanZero(key)); + } + + probe.SuccessThreshold = successThreshold; + break; + case "failureThreshold": + if (!int.TryParse(YamlParser.GetScalarValue(key, child.Value), out var failureThreshold)) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeAnInteger(key)); + } + + if (failureThreshold < 1) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeGreaterThanZero(key)); + } + + probe.FailureThreshold = failureThreshold; + break; + default: + throw new TyeYamlException(child.Key.Start, CoreStrings.FormatUnrecognizedKey(key)); + } + } + } + + private static void HandleServiceHttpProber(YamlMappingNode yamlMappingNode, ConfigHttpProber prober) + { + foreach (var child in yamlMappingNode.Children) + { + var key = YamlParser.GetScalarValue(child.Key); + + switch (key) + { + case "path": + prober.Path = YamlParser.GetScalarValue("path", child.Value); + break; + case "port": + if (!int.TryParse(YamlParser.GetScalarValue(key, child.Value), out var port)) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatMustBeAnInteger(key)); + } + + prober.Port = port; + break; + case "protocol": + prober.Path = YamlParser.GetScalarValue("protocol", child.Value); + break; + case "headers": + prober.Headers = new List>(); + var headersNode = child.Value as YamlSequenceNode; + if (headersNode is null) + { + throw new TyeYamlException(child.Value.Start, CoreStrings.FormatExpectedYamlSequence("headers")); + } + + foreach (var header in headersNode.Children) + { + HandleServiceProbeHttpHeader((YamlMappingNode)header, prober.Headers); + } + + break; + default: + throw new TyeYamlException(child.Key.Start, CoreStrings.FormatUnrecognizedKey(key)); + } + } + } + + private static void HandleServiceProbeHttpHeader(YamlMappingNode yamlMappingNode, List> headers) + { + string? name = null; + object? value = null; + + foreach (var child in yamlMappingNode.Children) + { + var key = YamlParser.GetScalarValue(child.Key); + + switch (key) + { + case "name": + name = YamlParser.GetScalarValue("name", child.Value); + break; + case "value": + value = YamlParser.GetScalarValue("value", child.Value); + break; + default: + throw new TyeYamlException(child.Key.Start, CoreStrings.FormatUnrecognizedKey(key)); + } + } + + if (name is null) + { + throw new TyeYamlException(yamlMappingNode.Start, CoreStrings.FormatExpectedYamlScalar("name")); + } + else if (value is null) + { + throw new TyeYamlException(yamlMappingNode.Start, CoreStrings.FormatExpectedYamlScalar("value")); + } + + headers.Add(new KeyValuePair(name, value)); + } + private static void HandleServiceVolumeNameMapping(YamlMappingNode yamlMappingNode, ConfigVolume volume) { foreach (var child in yamlMappingNode!.Children) diff --git a/src/Microsoft.Tye.Hosting/DockerRunner.cs b/src/Microsoft.Tye.Hosting/DockerRunner.cs index 49d53e7f5..6724f3529 100644 --- a/src/Microsoft.Tye.Hosting/DockerRunner.cs +++ b/src/Microsoft.Tye.Hosting/DockerRunner.cs @@ -248,6 +248,7 @@ async Task RunDockerContainer(IEnumerable<(int ExternalPort, int Port, int? Cont if (hasPorts) { status.Ports = ports.Select(p => p.Port); + status.Bindings = ports.Select(p => new ReplicaBinding() { Port = p.Port, ExternalPort = p.ExternalPort, Protocol = p.Protocol }).ToList(); // These are the ports that the application should use for binding @@ -327,7 +328,7 @@ async Task RunDockerContainer(IEnumerable<(int ExternalPort, int Port, int? Cont if (result.ExitCode != 0) { _logger.LogError("docker run failed for {ServiceName} with exit code {ExitCode}:" + result.StandardError, service.Description.Name, result.ExitCode); - service.Replicas.TryRemove(replica, out _); + service.Replicas.TryRemove(replica, out var _); service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status)); PrintStdOutAndErr(service, replica, result); @@ -367,42 +368,73 @@ async Task RunDockerContainer(IEnumerable<(int ExternalPort, int Port, int? Cont PrintStdOutAndErr(service, replica, result); } - service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status)); - - _logger.LogInformation("Collecting docker logs for {ContainerName}.", replica); - - var backOff = TimeSpan.FromSeconds(5); + var sentStartedEvent = false; while (!dockerInfo.StoppingTokenSource.Token.IsCancellationRequested) { - var logsRes = await ProcessUtil.RunAsync("docker", $"logs -f {containerId}", - outputDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"), - errorDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"), - throwOnError: false, - cancellationToken: dockerInfo.StoppingTokenSource.Token); - - if (logsRes.ExitCode != 0) + if (sentStartedEvent) { - break; + using var restartCts = new CancellationTokenSource(DockerStopTimeout); + result = await ProcessUtil.RunAsync("docker", $"restart {containerId}", throwOnError: false, cancellationToken: restartCts.Token); + + if (restartCts.IsCancellationRequested) + { + _logger.LogWarning($"Failed to restart container after {DockerStopTimeout.Seconds} seconds.", replica, shortContainerId); + break; // implement retry mechanism? + } + else if (result.ExitCode != 0) + { + _logger.LogWarning($"Failed to restart container due to exit code {result.ExitCode}.", replica, shortContainerId); + break; + } + + service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status)); } - if (!dockerInfo.StoppingTokenSource.IsCancellationRequested) + using var stoppingCts = new CancellationTokenSource(); + status.StoppingTokenSource = stoppingCts; + service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status)); + sentStartedEvent = true; + + await using var _ = dockerInfo.StoppingTokenSource.Token.Register(() => status.StoppingTokenSource.Cancel()); + + _logger.LogInformation("Collecting docker logs for {ContainerName}.", replica); + + var backOff = TimeSpan.FromSeconds(5); + + while (!status.StoppingTokenSource.Token.IsCancellationRequested) { - try + var logsRes = await ProcessUtil.RunAsync("docker", $"logs -f {containerId}", + outputDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"), + errorDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"), + throwOnError: false, + cancellationToken: status.StoppingTokenSource.Token); + + if (logsRes.ExitCode != 0) { - // Avoid spamming logs if restarts are happening - await Task.Delay(backOff, dockerInfo.StoppingTokenSource.Token); + break; } - catch (OperationCanceledException) + + if (!status.StoppingTokenSource.IsCancellationRequested) { - break; + try + { + // Avoid spamming logs if restarts are happening + await Task.Delay(backOff, status.StoppingTokenSource.Token); + } + catch (OperationCanceledException) + { + break; + } } + + backOff *= 2; } - backOff *= 2; - } + _logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode); - _logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode); + status.StoppingTokenSource = null; + } // Docker has a tendency to get stuck so we're going to timeout this shutdown process var timeoutCts = new CancellationTokenSource(DockerStopTimeout); @@ -418,7 +450,10 @@ async Task RunDockerContainer(IEnumerable<(int ExternalPort, int Port, int? Cont PrintStdOutAndErr(service, replica, result); - service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status)); + if (sentStartedEvent) + { + service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Stopped, status)); + } _logger.LogInformation("Stopped container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode); @@ -433,7 +468,7 @@ async Task RunDockerContainer(IEnumerable<(int ExternalPort, int Port, int? Cont _logger.LogInformation("Removed container {ContainerName} with ID {ContainerId} exited with {ExitCode}", replica, shortContainerId, result.ExitCode); - service.Replicas.TryRemove(replica, out _); + service.Replicas.TryRemove(replica, out var _); service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status)); }; diff --git a/src/Microsoft.Tye.Hosting/HttpProxyService.cs b/src/Microsoft.Tye.Hosting/HttpProxyService.cs index 8148ae73d..8b0b1386d 100644 --- a/src/Microsoft.Tye.Hosting/HttpProxyService.cs +++ b/src/Microsoft.Tye.Hosting/HttpProxyService.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net; @@ -26,9 +27,12 @@ public partial class HttpProxyService : IApplicationProcessor private List _webApplications = new List(); private readonly ILogger _logger; + private ConcurrentDictionary _readyPorts; + public HttpProxyService(ILogger logger) { _logger = logger; + _readyPorts = new ConcurrentDictionary(); } public async Task StartAsync(Application application) @@ -98,8 +102,9 @@ public async Task StartAsync(Application application) _logger.LogInformation("Processing ingress rule: Path:{Path}, Host:{Host}, Service:{Service}", rule.Path, rule.Host, rule.Service); var targetServiceDescription = target.Description; + RegisterListener(target); - var uris = new List(); + var uris = new List<(int Port, Uri Uri)>(); // HTTP before HTTPS (this might change once we figure out certs...) var targetBinding = targetServiceDescription.Bindings.FirstOrDefault(b => b.Protocol == "http") ?? @@ -117,23 +122,42 @@ public async Task StartAsync(Application application) { var port = targetBinding.ReplicaPorts[i]; var url = $"{targetBinding.Protocol}://localhost:{port}"; - uris.Add(new Uri(url)); + uris.Add((port, new Uri(url))); } _logger.LogInformation("Service {ServiceName} is using {Urls}", targetServiceDescription.Name, string.Join(",", uris.Select(u => u.ToString()))); // The only load balancing strategy here is round robin long count = 0; - RequestDelegate del = context => + RequestDelegate del = async context => { var next = (int)(Interlocked.Increment(ref count) % uris.Count); - var uri = new UriBuilder(uris[next]) + // we find the first `Ready` port + for (int i = 0; i < uris.Count; i++) + { + if (_readyPorts.ContainsKey(uris[next].Port)) + { + break; + } + + next = (int)(Interlocked.Increment(ref count) % uris.Count); + } + + // if we've looped through all the port and didn't find a single one that is `Ready`, we return HTTP BadGateway + if (!_readyPorts.ContainsKey(uris[next].Port)) + { + context.Response.StatusCode = (int)HttpStatusCode.BadGateway; + await context.Response.WriteAsync("Bad gateway"); + return; + } + + var uri = new UriBuilder(uris[next].Uri) { Path = (string)context.Request.RouteValues["path"] }; - return context.ProxyRequest(invoker, uri.Uri); + await context.ProxyRequest(invoker, uri.Uri); }; IEndpointConventionBuilder conventions = null!; @@ -167,6 +191,14 @@ public async Task StartAsync(Application application) public async Task StopAsync(Application application) { + foreach (var service in application.Services.Values) + { + if (service.Items.TryGetValue(typeof(Subscription), out var item) && item is IDisposable disposable) + { + disposable.Dispose(); + } + } + foreach (var webApp in _webApplications) { try @@ -183,5 +215,32 @@ public async Task StopAsync(Application application) } } } + + private void RegisterListener(Service service) + { + if (!service.Items.ContainsKey(typeof(Subscription))) + { + service.Items[typeof(Subscription)] = service.ReplicaEvents.Subscribe(OnReplicaEvent); + } + } + + private void OnReplicaEvent(ReplicaEvent replicaEvent) + { + foreach (var binding in replicaEvent.Replica.Bindings) + { + if (replicaEvent.State == ReplicaState.Ready) + { + _readyPorts.TryAdd(binding.Port, true); + } + else + { + _readyPorts.TryRemove(binding.Port, out _); + } + } + } + + private class Subscription + { + } } } diff --git a/src/Microsoft.Tye.Hosting/Model/HttpProber.cs b/src/Microsoft.Tye.Hosting/Model/HttpProber.cs new file mode 100644 index 000000000..8eb642ef8 --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Model/HttpProber.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; + +namespace Microsoft.Tye.Hosting.Model +{ + public class HttpProber + { + public string Path { get; set; } = default!; + public int? Port { get; set; } + public string? Protocol { get; set; } + public List> Headers { get; set; } = new List>(); + } +} diff --git a/src/Microsoft.Tye.Hosting/Model/Probe.cs b/src/Microsoft.Tye.Hosting/Model/Probe.cs new file mode 100644 index 000000000..1e36f01ac --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Model/Probe.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.Tye.Hosting.Model +{ + public class Probe + { + public HttpProber? Http { get; set; } + public TimeSpan InitialDelay { get; set; } + public TimeSpan Period { get; set; } + public TimeSpan Timeout { get; set; } + public int SuccessThreshold { get; set; } + public int FailureThreshold { get; set; } + } +} diff --git a/src/Microsoft.Tye.Hosting/Model/ReplicaBinding.cs b/src/Microsoft.Tye.Hosting/Model/ReplicaBinding.cs new file mode 100644 index 000000000..e5aaba503 --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Model/ReplicaBinding.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.Tye.Hosting.Model +{ + public class ReplicaBinding + { + public int Port { get; set; } + public int ExternalPort { get; set; } + public string? Protocol { get; set; } + } +} diff --git a/src/Microsoft.Tye.Hosting/Model/ReplicaState.cs b/src/Microsoft.Tye.Hosting/Model/ReplicaState.cs index 5e08bd9f7..0b5b826b3 100644 --- a/src/Microsoft.Tye.Hosting/Model/ReplicaState.cs +++ b/src/Microsoft.Tye.Hosting/Model/ReplicaState.cs @@ -10,5 +10,7 @@ public enum ReplicaState Added, Started, Stopped, + Healthy, + Ready } } diff --git a/src/Microsoft.Tye.Hosting/Model/ReplicaStatus.cs b/src/Microsoft.Tye.Hosting/Model/ReplicaStatus.cs index eee5a084f..c8cd2a2bc 100644 --- a/src/Microsoft.Tye.Hosting/Model/ReplicaStatus.cs +++ b/src/Microsoft.Tye.Hosting/Model/ReplicaStatus.cs @@ -2,8 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Threading; namespace Microsoft.Tye.Hosting.Model { @@ -26,5 +28,11 @@ public ReplicaStatus(Service service, string name) public ConcurrentDictionary Metrics { get; set; } = new ConcurrentDictionary(); public IDictionary? Environment { get; set; } + + public ReplicaState? State { get; set; } + + public CancellationTokenSource? StoppingTokenSource { get; set; } + + public List Bindings { get; set; } = new List(); } } diff --git a/src/Microsoft.Tye.Hosting/Model/Service.cs b/src/Microsoft.Tye.Hosting/Model/Service.cs index 038622582..cdf5baf0c 100644 --- a/src/Microsoft.Tye.Hosting/Model/Service.cs +++ b/src/Microsoft.Tye.Hosting/Model/Service.cs @@ -24,6 +24,11 @@ public Service(ServiceDescription description) CachedLogs.Enqueue(entry); }); + + ReplicaEvents.Subscribe(entry => + { + entry.Replica.State = entry.State; + }); } public ServiceDescription Description { get; } diff --git a/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs b/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs index 52df9f69d..096c991ff 100644 --- a/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs +++ b/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs @@ -20,5 +20,7 @@ public ServiceDescription(string name, RunInfo? runInfo) public List Bindings { get; } = new List(); public List Configuration { get; } = new List(); public List Dependencies { get; } = new List(); + public Probe? Liveness { get; set; } + public Probe? Readiness { get; set; } } } diff --git a/src/Microsoft.Tye.Hosting/Model/V1/V1ReplicaStatus.cs b/src/Microsoft.Tye.Hosting/Model/V1/V1ReplicaStatus.cs index 61e3861a6..83d0078f3 100644 --- a/src/Microsoft.Tye.Hosting/Model/V1/V1ReplicaStatus.cs +++ b/src/Microsoft.Tye.Hosting/Model/V1/V1ReplicaStatus.cs @@ -18,5 +18,6 @@ public class V1ReplicaStatus public int? ExitCode { get; set; } public int? Pid { get; set; } public IDictionary? Environment { get; set; } + public ReplicaState? State { get; set; } } } diff --git a/src/Microsoft.Tye.Hosting/PortAssigner.cs b/src/Microsoft.Tye.Hosting/PortAssigner.cs index d5a1c670f..2e1885806 100644 --- a/src/Microsoft.Tye.Hosting/PortAssigner.cs +++ b/src/Microsoft.Tye.Hosting/PortAssigner.cs @@ -50,7 +50,7 @@ static int GetNextPort() binding.Port = GetNextPort(); } - if (service.Description.Replicas == 1) + if (service.Description.Readiness == null && service.Description.Replicas == 1) { // No need to proxy, the port maps to itself binding.ReplicaPorts.Add(binding.Port.Value); diff --git a/src/Microsoft.Tye.Hosting/ProcessRunner.cs b/src/Microsoft.Tye.Hosting/ProcessRunner.cs index 30eb38aba..da731591d 100644 --- a/src/Microsoft.Tye.Hosting/ProcessRunner.cs +++ b/src/Microsoft.Tye.Hosting/ProcessRunner.cs @@ -238,6 +238,10 @@ async Task RunApplicationAsync(IEnumerable<(int ExternalPort, int Port, string? var status = new ProcessStatus(service, replica); service.Replicas[replica] = status; + using var stoppingCts = new CancellationTokenSource(); + status.StoppingTokenSource = stoppingCts; + await using var _ = processInfo.StoppedTokenSource.Token.Register(() => status.StoppingTokenSource.Cancel()); + service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status)); // This isn't your host name @@ -250,6 +254,7 @@ async Task RunApplicationAsync(IEnumerable<(int ExternalPort, int Port, string? if (hasPorts) { status.Ports = ports.Select(p => p.Port); + status.Bindings = ports.Select(p => new ReplicaBinding() { Port = p.Port, ExternalPort = p.ExternalPort, Protocol = p.Protocol }).ToList(); } // TODO clean this up. @@ -291,7 +296,7 @@ async Task RunApplicationAsync(IEnumerable<(int ExternalPort, int Port, string? service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Started, status)); }, throwOnError: false, - cancellationToken: processInfo.StoppedTokenSource.Token); + cancellationToken: status.StoppingTokenSource.Token); status.ExitCode = result.ExitCode; @@ -324,7 +329,7 @@ async Task RunApplicationAsync(IEnumerable<(int ExternalPort, int Port, string? } // Remove the replica from the set - service.Replicas.TryRemove(replica, out _); + service.Replicas.TryRemove(replica, out var _); service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Removed, status)); } } diff --git a/src/Microsoft.Tye.Hosting/ProxyService.cs b/src/Microsoft.Tye.Hosting/ProxyService.cs index 2723c1b24..a78fa49a5 100644 --- a/src/Microsoft.Tye.Hosting/ProxyService.cs +++ b/src/Microsoft.Tye.Hosting/ProxyService.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections.Concurrent; using System.IO; using System.IO.Pipelines; using System.Net; @@ -13,7 +14,6 @@ using Bedrock.Framework; using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Connections.Features; -using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Tye.Hosting.Model; @@ -25,9 +25,12 @@ public class ProxyService : IApplicationProcessor private IHost? _host; private readonly ILogger _logger; + private ConcurrentDictionary _cancellationsByReplicaPort; + public ProxyService(ILogger logger) { _logger = logger; + _cancellationsByReplicaPort = new ConcurrentDictionary(); } public Task StartAsync(Application application) @@ -44,6 +47,8 @@ public Task StartAsync(Application application) continue; } + service.Items[typeof(Subscription)] = service.ReplicaEvents.Subscribe(OnReplicaEvent); + foreach (var binding in service.Description.Bindings) { if (binding.Port == null) @@ -52,7 +57,7 @@ public Task StartAsync(Application application) continue; } - if (service.Description.Replicas == 1) + if (service.Description.Readiness == null && service.Description.Replicas == 1) { // No need to proxy for a single replica, we may want to do this later but right now we skip it continue; @@ -77,6 +82,15 @@ public Task StartAsync(Application application) var next = (int)(Interlocked.Increment(ref count) % ports.Count); + if (!_cancellationsByReplicaPort.TryGetValue(ports[next], out var cts)) + { + // replica in ready state <=> it's ports have cancellation tokens in the dictionary + // if replica is not in ready state, we don't forward traffic, but return instead + return; + } + + using var _ = cts.Token.Register(() => notificationFeature.RequestClose()); + NetworkStream? targetStream = null; try @@ -134,6 +148,8 @@ public Task StartAsync(Application application) { _logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", service.Description.Name); } + + _logger.LogDebug("Existing proxy {ServiceName} {ExternalPort}:{InternalPort}", service.Description.Name, binding.Port, ports[next]); } catch (Exception ex) { @@ -159,6 +175,14 @@ public Task StartAsync(Application application) public async Task StopAsync(Application application) { + foreach (var service in application.Services.Values) + { + if (service.Items.TryGetValue(typeof(Subscription), out var item) && item is IDisposable disposable) + { + disposable.Dispose(); + } + } + if (_host != null) { await _host.StopAsync(); @@ -169,5 +193,30 @@ public async Task StopAsync(Application application) } } } + + private void OnReplicaEvent(ReplicaEvent replicaEvent) + { + // when a replica becomes ready for the first time, it shouldn't have a cancellation token in the dictionary + // for any event other than ready, we want to cancel the token and remove it from the dictionary + foreach (var binding in replicaEvent.Replica.Bindings) + { + if (_cancellationsByReplicaPort.TryRemove(binding.Port, out var cts)) + { + cts.Cancel(); + } + } + + if (replicaEvent.State == ReplicaState.Ready) + { + foreach (var binding in replicaEvent.Replica.Bindings) + { + _cancellationsByReplicaPort.TryAdd(binding.Port, new CancellationTokenSource()); + } + } + } + + private class Subscription + { + } } } diff --git a/src/Microsoft.Tye.Hosting/ReplicaMonitor.cs b/src/Microsoft.Tye.Hosting/ReplicaMonitor.cs new file mode 100644 index 000000000..0ea8138fa --- /dev/null +++ b/src/Microsoft.Tye.Hosting/ReplicaMonitor.cs @@ -0,0 +1,455 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Net.Http; +using System.Reactive.Subjects; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Tye.Hosting.Model; + +namespace Microsoft.Tye.Hosting +{ + public class ReplicaMonitor : IApplicationProcessor + { + private ILogger _logger; + private ConcurrentDictionary _states; + + public ReplicaMonitor(ILogger logger) + { + _logger = logger; + _states = new ConcurrentDictionary(); + } + + public Task StartAsync(Application application) + { + foreach (var service in application.Services.Values) + { + service.Items[typeof(Subscription)] = service.ReplicaEvents.Subscribe(OnReplicaChanged); + } + + return Task.CompletedTask; + } + + public Task StopAsync(Application application) + { + foreach (var service in application.Services.Values) + { + if (service.Items.TryGetValue(typeof(Subscription), out var item) && item is IDisposable disposable) + { + disposable.Dispose(); + } + } + + return Task.CompletedTask; + } + + private void OnReplicaChanged(ReplicaEvent replicaEvent) + { + switch (replicaEvent.State) + { + case ReplicaState.Started: + _states.TryAdd(replicaEvent.Replica.Name, new ReplicaMonitorState(replicaEvent.Replica, _logger)); + break; + case ReplicaState.Stopped: + if (_states.TryRemove(replicaEvent.Replica.Name, out var stateToDispose)) + { + stateToDispose.Dispose(); + } + break; + default: + if (_states.TryGetValue(replicaEvent.Replica.Name, out var state)) + { + state.Update(replicaEvent); + } + break; + } + } + + private class Subscription + { + } + + private class ReplicaMonitorState : IDisposable + { + private ReplicaStatus _replica; + private ILogger _logger; + + private Prober? _livenessProber; + private Prober? _readinessProber; + private IDisposable? _livenessProberObserver; + private IDisposable? _readinessProberObserver; + + private ReplicaState _currentState; + private DateTime _lastStateChange; + private object _stateChangeLocker; + + public ReplicaMonitorState(ReplicaStatus replica, ILogger logger) + { + _replica = replica; + _logger = logger; + + _currentState = ReplicaState.Started; + _lastStateChange = DateTime.Now; + _stateChangeLocker = new object(); + + Init(); + } + + private void Init() + { + var serviceDesc = _replica.Service.Description; + if (serviceDesc.Liveness == null && serviceDesc.Readiness == null) + { + MoveToReady(); + } + else if (serviceDesc.Liveness == null) + { + MoveToHealthy(from: ReplicaState.Started); + StartReadinessProbe(serviceDesc.Readiness!); + } + else if (serviceDesc.Readiness == null) + { + StartLivenessProbe(serviceDesc.Liveness, moveToOnSuccess: ReplicaState.Ready); + } + else + { + StartLivenessProbe(serviceDesc.Liveness); + StartReadinessProbe(serviceDesc.Readiness); + } + } + + private void StartLivenessProbe(Probe probe, ReplicaState moveToOnSuccess = ReplicaState.Healthy) + { + // currently only HTTP is available + if (probe.Http == null) + { + _logger.LogWarning("Cannot start probing replica {name} because probe configuration is not set", _replica.Name); + return; + } + + _livenessProber = new HttpProber(_replica, "liveness", probe, probe.Http, _logger); + + var failureThreshold = probe.FailureThreshold; + var failures = 0; + var dead = false; + + _livenessProberObserver = _livenessProber.ProbeResults.Subscribe(entry => + { + if (dead) + { + return; + } + + if (entry) + { + // Reset failures count on success + failures = 0; + } + + (var currentState, _) = ReadCurrentState(); + var failuresPastThreshold = failures >= failureThreshold; + switch ((entry, currentState, moveToOnSuccess, failuresPastThreshold)) + { + case (false, _, _, true): + dead = true; + Kill(); + break; + case (false, _, _, false): + Interlocked.Increment(ref failures); + break; + case (true, ReplicaState.Started, ReplicaState.Ready, _): + case (true, ReplicaState.Healthy, ReplicaState.Ready, _): + MoveToReady(); + break; + case (true, ReplicaState.Started, ReplicaState.Healthy, _): + MoveToHealthy(from: ReplicaState.Started); + break; + } + }); + + _livenessProber.Start(); + } + + private void StartReadinessProbe(Probe probe) + { + // currently only HTTP is available + if (probe.Http == null) + { + _logger.LogWarning("Cannot start probing replica {name} because probe configuration is not set", _replica.Name); + return; + } + + _readinessProber = new HttpProber(_replica, "readiness", probe, probe.Http, _logger); + + var successThreshold = probe.SuccessThreshold; + var failureThreshold = probe.FailureThreshold; + + var successes = 0; + var failures = 0; + + _readinessProberObserver = _readinessProber.ProbeResults.Subscribe(entry => + { + if (entry) + { + // Reset failures count on success + failures = 0; + } + else + { + // Reset successes count on failure + successes = 0; + } + + (var currentState, _) = ReadCurrentState(); + var successesPastThreshold = successes >= successThreshold; + var failuresPastThreshold = failures >= failureThreshold; + switch ((entry, currentState, failuresPastThreshold, successesPastThreshold)) + { + case (false, ReplicaState.Ready, true, _): + MoveToHealthy(from: ReplicaState.Ready); + break; + case (false, ReplicaState.Ready, false, _): + Interlocked.Increment(ref failures); + break; + case (true, ReplicaState.Healthy, _, true): + MoveToReady(); + break; + case (true, ReplicaState.Healthy, _, false): + Interlocked.Increment(ref successes); + break; + } + }); + + _readinessProber.Start(); + } + + private void MoveToHealthy(ReplicaState from) + { + _logger.LogDebug("Replica {name} is moving to an healthy state", _replica.Name); + ChangeState(ReplicaState.Healthy); + } + + private void MoveToReady() + { + _logger.LogDebug("Replica {name} is moving to a ready state", _replica.Name); + ChangeState(ReplicaState.Ready); + } + + private void Kill() + { + _logger.LogDebug("Killing replica {name} because it has failed the liveness probe", _replica.Name); + + // it is assumed that a `Started` replica should have an initialized stopping token source + _replica.StoppingTokenSource!.Cancel(); + } + + private void ChangeState(ReplicaState state) + { + _replica.Service.ReplicaEvents.OnNext(new ReplicaEvent(state, _replica)); + lock (_stateChangeLocker) + { + _currentState = state; + _lastStateChange = DateTime.Now; + } + } + + private (ReplicaState state, DateTime lastChanged) ReadCurrentState() + { + lock (_stateChangeLocker) + { + return (_currentState, _lastStateChange); + } + } + + public void Update(ReplicaEvent replicaEvent) + { + } + + public void Dispose() + { + _livenessProber?.Dispose(); + _readinessProber?.Dispose(); + _livenessProberObserver?.Dispose(); + _readinessProberObserver?.Dispose(); + } + } + + private abstract class Prober : IDisposable + { + protected Prober() + { + ProbeResults = new Subject(); + } + + public Subject ProbeResults { get; } + + public abstract void Start(); + + public abstract void Dispose(); + } + + private class HttpProber : Prober + { + private static HttpClient _httpClient; + + static HttpProber() + { + _httpClient = new HttpClient(); + } + + private ReplicaStatus _replica; + private ReplicaBinding? _selectedBinding; + private string _probeName; + private Probe _probe; + private Model.HttpProber _httpProberSettings; + + private Timer _probeTimer; + private CancellationTokenSource _cts; + + private ILogger _logger; + + private bool _lastStatus; + + public HttpProber(ReplicaStatus replica, string probeName, Probe probe, Model.HttpProber httpProberSettings, ILogger logger) + : base() + { + _replica = replica; + _selectedBinding = null; + _probeName = probeName; + _probe = probe; + _httpProberSettings = httpProberSettings; + + _probeTimer = new Timer(DoProbe, null, Timeout.Infinite, Timeout.Infinite); + _cts = new CancellationTokenSource(); + + _logger = logger; + + _lastStatus = true; + } + + private void DoProbe(object? state) + { + _probeTimer.Change(Timeout.Infinite, Timeout.Infinite); + _ = DoProbeAsync(); + } + + private async Task DoProbeAsync() + { + if (_cts.Token.IsCancellationRequested) + { + return; + } + + try + { + var protocol = _selectedBinding!.Protocol; + var address = $"{protocol}://localhost:{_selectedBinding.Port}{_httpProberSettings.Path}"; + + using var timeoutCts = new CancellationTokenSource(_probe.Timeout); + var req = new HttpRequestMessage(HttpMethod.Get, address); + foreach (var header in _httpProberSettings.Headers) + { + req.Headers.Add(header.Key, header.Value.ToString()); + } + + var res = await _httpClient.SendAsync(req, timeoutCts.Token); + if (!res.IsSuccessStatusCode) + { + ShowWarning($"Replica {_replica.Name} failed http probe at address '{_httpProberSettings.Path}' due to a failed status ({res.StatusCode})"); + Send(false); + return; + } + + Send(true); + } + catch (HttpRequestException ex) + { + ShowWarning($"Replica {_replica.Name} failed http probe at address '{_httpProberSettings.Path}' due to an http exception", ex); + Send(false); + } + catch (TaskCanceledException) + { + ShowWarning($"Replica {_replica.Name} failed http probe at address '{_httpProberSettings.Path}' due to timeout"); + Send(false); + } + finally + { + try + { + _probeTimer.Change(_probe.Period, Timeout.InfiniteTimeSpan); + } + catch (ObjectDisposedException) + { + } + } + } + + private void Send(bool status) + { + ProbeResults.OnNext(status); + _lastStatus = status; + } + + private void ShowWarning(string message, Exception? ex = null) + { + if (!_lastStatus) + { + return; + } + + if (ex != null) + { + _logger.LogWarning(ex, message); + } + else + { + _logger.LogWarning(message); + } + } + + public override void Start() + { + // the logic that selects the binding for the probing depends on the protocol and port fields that were provided in the probe + // if neither port nor protocol were provided, we select the first binding + // if just port was provided, we select the first binding with that port + // if just protocol was provided, we select the first binding with that protocol (http/https) + // if both port and protocol were provided, we select the first binding with that port and protocol + Func bindingClosure = (_httpProberSettings.Port.HasValue, _httpProberSettings.Protocol != null) switch + { + (false, false) => _ => true, + (true, false) => r => r.ExternalPort == _httpProberSettings.Port!.Value, + (false, true) => r => r.Protocol == _httpProberSettings.Protocol!, + (true, true) => r => r.ExternalPort == _httpProberSettings.Port!.Value && r.Protocol == _httpProberSettings.Protocol! + }; + + var selectedBindings = _replica.Bindings.Where(bindingClosure); + if (selectedBindings.Count() == 0) + { + _logger.LogWarning($"No suitable binding was found for replica {_replica.Name} for probe '{_probeName}'"); + return; + } + + _selectedBinding = selectedBindings.First(); + + try + { + _probeTimer.Change(_probe.InitialDelay, Timeout.InfiniteTimeSpan); + } + catch (ObjectDisposedException) + { + } + } + + public override void Dispose() + { + _cts.Cancel(); + _probeTimer.Dispose(); + } + } + } +} diff --git a/src/Microsoft.Tye.Hosting/TyeDashboardApi.cs b/src/Microsoft.Tye.Hosting/TyeDashboardApi.cs index caf7780cd..48b707cdf 100644 --- a/src/Microsoft.Tye.Hosting/TyeDashboardApi.cs +++ b/src/Microsoft.Tye.Hosting/TyeDashboardApi.cs @@ -171,7 +171,8 @@ private static V1Service CreateServiceJson(Service service) { Name = replica.Value.Name, Ports = replica.Value.Ports, - Environment = replica.Value.Environment + Environment = replica.Value.Environment, + State = replica.Value.State }; replicateDictionary[replica.Key] = replicaStatus; diff --git a/src/Microsoft.Tye.Hosting/TyeHost.cs b/src/Microsoft.Tye.Hosting/TyeHost.cs index f5b6b5feb..11838a61b 100644 --- a/src/Microsoft.Tye.Hosting/TyeHost.cs +++ b/src/Microsoft.Tye.Hosting/TyeHost.cs @@ -276,6 +276,7 @@ private static AggregateApplicationProcessor CreateApplicationProcessor(ReplicaR new ProxyService(logger), new HttpProxyService(logger), new DockerImagePuller(logger), + new ReplicaMonitor(logger), new DockerRunner(logger, replicaRegistry), new ProcessRunner(logger, replicaRegistry, ProcessRunnerOptions.FromHostOptions(options)) }; diff --git a/src/tye/ApplicationBuilderExtensions.cs b/src/tye/ApplicationBuilderExtensions.cs index dae5de1fd..6e5786e6e 100644 --- a/src/tye/ApplicationBuilderExtensions.cs +++ b/src/tye/ApplicationBuilderExtensions.cs @@ -36,11 +36,15 @@ public static Application ToHostingApplication(this ApplicationBuilder applicati foreach (var service in application.Services) { RunInfo? runInfo; + Probe? liveness; + Probe? readiness; int replicas; var env = new List(); if (service is ExternalServiceBuilder) { runInfo = null; + liveness = null; + readiness = null; replicas = 1; } else if (service is ContainerServiceBuilder container) @@ -70,6 +74,8 @@ public static Application ToHostingApplication(this ApplicationBuilder applicati runInfo = dockerRunInfo; replicas = container.Replicas; + liveness = container.Liveness != null ? GetProbeFromBuilder(container.Liveness) : null; + readiness = container.Readiness != null ? GetProbeFromBuilder(container.Readiness) : null; foreach (var entry in container.EnvironmentVariables) { @@ -80,6 +86,8 @@ public static Application ToHostingApplication(this ApplicationBuilder applicati { runInfo = new ExecutableRunInfo(executable.Executable, executable.WorkingDirectory, executable.Args); replicas = executable.Replicas; + liveness = executable.Liveness != null ? GetProbeFromBuilder(executable.Liveness) : null; + readiness = executable.Readiness != null ? GetProbeFromBuilder(executable.Readiness) : null; foreach (var entry in executable.EnvironmentVariables) { @@ -107,6 +115,8 @@ public static Application ToHostingApplication(this ApplicationBuilder applicati runInfo = projectInfo; replicas = project.Replicas; + liveness = project.Liveness != null ? GetProbeFromBuilder(project.Liveness) : null; + readiness = project.Readiness != null ? GetProbeFromBuilder(project.Readiness) : null; foreach (var entry in project.EnvironmentVariables) { @@ -121,6 +131,8 @@ public static Application ToHostingApplication(this ApplicationBuilder applicati var description = new ServiceDescription(service.Name, runInfo) { Replicas = replicas, + Liveness = liveness, + Readiness = readiness }; description.Configuration.AddRange(env); @@ -187,5 +199,21 @@ public static Tye.Hosting.Model.EnvironmentVariable ToHostingEnvironmentVariable return env; } + + private static Probe GetProbeFromBuilder(ProbeBuilder builder) => new Probe() + { + Http = builder.Http != null ? new HttpProber() + { + Path = builder.Http.Path, + Headers = builder.Http.Headers, + Port = builder.Http.Port, + Protocol = builder.Http.Protocol + } : null, + InitialDelay = TimeSpan.FromSeconds(builder.InitialDelay), + Period = TimeSpan.FromSeconds(builder.Period), + Timeout = TimeSpan.FromSeconds(builder.Timeout), + SuccessThreshold = builder.SuccessThreshold, + FailureThreshold = builder.FailureThreshold + }; } } diff --git a/test/E2ETest/HealthCheckTests.cs b/test/E2ETest/HealthCheckTests.cs new file mode 100644 index 000000000..288e48c6d --- /dev/null +++ b/test/E2ETest/HealthCheckTests.cs @@ -0,0 +1,453 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Microsoft.Tye; +using Microsoft.Tye.Hosting; +using Microsoft.Tye.Hosting.Model; +using Test.Infrastructure; +using Xunit; +using Xunit.Abstractions; +using static Test.Infrastructure.TestHelpers; + +namespace E2ETest +{ + public class HealthCheckTests + { + private readonly ITestOutputHelper _output; + private readonly TestOutputLogEventSink _sink; + private readonly JsonSerializerOptions _options; + + private static readonly ReplicaState?[] startedOrHigher = new ReplicaState?[] { ReplicaState.Started, ReplicaState.Healthy, ReplicaState.Ready }; + private static readonly ReplicaState?[] stoppedOrLower = new ReplicaState?[] { ReplicaState.Stopped, ReplicaState.Removed }; + + private static HttpClient _client; + + static HealthCheckTests() + { + var handler = new HttpClientHandler + { + ServerCertificateCustomValidationCallback = (a, b, c, d) => true, + AllowAutoRedirect = false + }; + + _client = new HttpClient(new RetryHandler(handler)); + } + + public HealthCheckTests(ITestOutputHelper output) + { + _output = output; + _sink = new TestOutputLogEventSink(output); + + _options = new JsonSerializerOptions() + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + }; + + _options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + } + + [Fact] + public async Task ServiceWithoutLivenessReadinessShouldDefaultToReadyTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-none.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-none" }, ReplicaState.Ready); + } + + [Fact] + public async Task ServiceWithoutLivenessShouldDefaultToHealthyTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-readiness.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-readiness" }, ReplicaState.Healthy); + } + + [Fact] + public async Task ServicWithoutLivenessShouldBecomeReadyWhenReadyTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-readiness.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-readiness" }, ReplicaState.Healthy); + + var replicas = host.Application.Services["health-readiness"].Replicas.Select(r => r.Value).ToList(); + Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, replicas.Count, replicas.Select(r => r.Name).ToHashSet(), null, TimeSpan.Zero, async _ => + { + await Task.WhenAll(replicas.Select(r => SetHealthyReadyInReplica(r, ready: true))); + })); + } + + [Fact] + public async Task ServiceWithoutReadinessShouldDefaultToReadyWhenHealthyTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-liveness.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-liveness" }, ReplicaState.Started); + + var replicasToBecomeReady = host.Application.Services["health-liveness"].Replicas.Select(r => r.Value); + var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); + Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, replicasNamesToBecomeReady.Count, replicasNamesToBecomeReady, null, TimeSpan.Zero, async _ => + { + await Task.WhenAll(replicasToBecomeReady.Select(r => SetHealthyReadyInReplica(r, healthy: true))); + })); + } + + [Fact] + public async Task ReadyServiceShouldBecomeHealthyWhenReadinessFailsTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + SetReplicasInitialState(host, true, true); + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); + + var replicasToBecomeReady = host.Application.Services["health-all"].Replicas.Select(r => r.Value).ToList(); + var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); + + var randomReplica = replicasToBecomeReady[new Random().Next(0, replicasToBecomeReady.Count)]; + + Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica.Name }.ToHashSet(), replicasNamesToBecomeReady.Where(r => r != randomReplica.Name).ToHashSet(), TimeSpan.FromSeconds(1), async _ => + { + await SetHealthyReadyInReplica(randomReplica, ready: false); + })); + } + + [Fact] + public async Task ReadyServiceShouldRestartWhenLivenessFailsTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + SetReplicasInitialState(host, true, true); + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); + + var replicasToBecomeReady = host.Application.Services["health-all"].Replicas.Select(r => r.Value).ToList(); + var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); + + var randomReplica = replicasToBecomeReady[new Random().Next(0, replicasToBecomeReady.Count)]; + + Assert.True(await DoOperationAndWaitForReplicasToRestart(host, new[] { randomReplica.Name }.ToHashSet(), replicasNamesToBecomeReady.Where(r => r != randomReplica.Name).ToHashSet(), TimeSpan.FromSeconds(1), async _ => + { + await SetHealthyReadyInReplica(randomReplica, healthy: false); + })); + } + + [Fact] + public async Task ProbeShouldRespectTimeoutTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + SetReplicasInitialState(host, true, true); + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); + + var replicasToBecomeReady = host.Application.Services["health-all"].Replicas.Select(r => r.Value).ToList(); + var replicasNamesToBecomeReady = replicasToBecomeReady.Select(r => r.Name).ToHashSet(); + + var randomNumber = new Random().Next(0, replicasToBecomeReady.Count); + var randomReplica1 = replicasToBecomeReady[randomNumber]; + var randomReplica2 = replicasToBecomeReady[(randomNumber + 1) % replicasToBecomeReady.Count]; + + Assert.True(await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica1.Name }.ToHashSet(), replicasNamesToBecomeReady.Where(r => r != randomReplica1.Name).ToHashSet(), TimeSpan.FromSeconds(2), async _ => + { + await Task.WhenAll(new[] + { + SetHealthyReadyInReplica(randomReplica1, readyDelay: 2), + SetHealthyReadyInReplica(randomReplica2, readyDelay: 1) + }); + })); + } + + [Fact] + public async Task ProxyShouldNotProxyToNonReadyReplicasTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-proxy.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + SetReplicasInitialState(host, true, true); + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-proxy" }, ReplicaState.Ready); + + var replicasToBecomeReady = host.Application.Services["health-proxy"].Replicas.Select(r => r.Value).ToList(); + + // we assume that proxy will continue sending http request to the same replica + var randomReplicaPortRes1 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); + var randomReplicaPort1 = JsonSerializer.Deserialize(await randomReplicaPortRes1.Content.ReadAsStringAsync())[0]; + var randomReplica1 = replicasToBecomeReady.First(r => r.Bindings.Any(b => b.Port == randomReplicaPort1)); + + await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica1.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => + { + await SetHealthyReadyInReplica(randomReplica1, ready: false); + }); + + var randomReplicaPortRes2 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); + var randomReplicaPort2 = JsonSerializer.Deserialize(await randomReplicaPortRes2.Content.ReadAsStringAsync())[0]; + var randomReplica2 = replicasToBecomeReady.First(r => r.Bindings.Any(b => b.Port == randomReplicaPort2)); + + Assert.NotEqual(randomReplicaPort1, randomReplicaPort2); + + await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica2.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => + { + await SetHealthyReadyInReplica(randomReplica2, ready: false); + }); + + try + { + var resShouldFail = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); + Assert.False(resShouldFail.IsSuccessStatusCode); + } + catch (HttpRequestException) + { + } + + await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, 1, new[] { randomReplica2.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => + { + await SetHealthyReadyInReplica(randomReplica2, ready: true); + }); + + var randomReplicaPortRes3 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); + var randomReplicaPort3 = JsonSerializer.Deserialize(await randomReplicaPortRes3.Content.ReadAsStringAsync())[0]; + + Assert.Equal(randomReplicaPort3, randomReplicaPort2); + } + + [Fact] + public async Task IngressShouldNotProxyToNonReadyReplicasTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-ingress.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + SetReplicasInitialState(host, true, true); + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-ingress-svc" }, ReplicaState.Ready); + + var replicasToBecomeReady = host.Application.Services["health-ingress-svc"].Replicas.Select(r => r.Value).ToList(); + var ingressBinding = host.Application.Services.First(s => s.Value.Description.RunInfo is IngressRunInfo).Value.Description.Bindings.First(); + var uniqueIdUrl = $"{ingressBinding.Protocol}://localhost:{ingressBinding.Port}/api/id"; + + var uniqueIds = await ProbeNumberOfUniqueReplicas(uniqueIdUrl); + Assert.Equal(2, uniqueIds); + + var firstReplica = replicasToBecomeReady.First(); + var secondReplica = replicasToBecomeReady.Skip(1).First(); + + await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { firstReplica.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => + { + await SetHealthyReadyInReplica(firstReplica, ready: false); + }); + + uniqueIds = await ProbeNumberOfUniqueReplicas(uniqueIdUrl); + Assert.Equal(1, uniqueIds); + + await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { secondReplica.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => + { + await SetHealthyReadyInReplica(secondReplica, ready: false); + }); + + var res = await _client.GetAsync(uniqueIdUrl); + Assert.Equal(HttpStatusCode.BadGateway, res.StatusCode); + + await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Ready, 2, new[] { firstReplica.Name, secondReplica.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => + { + await SetHealthyReadyInReplica(firstReplica, ready: true); + await SetHealthyReadyInReplica(secondReplica, ready: true); + }); + + uniqueIds = await ProbeNumberOfUniqueReplicas(uniqueIdUrl); + Assert.Equal(2, uniqueIds); + } + + [Fact] + public async Task HeadersTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await using var host = new TyeHost(application.ToHostingApplication(), new HostOptions()) + { + Sink = _sink, + }; + + SetReplicasInitialState(host, true, true); + + await StartHostAndWaitForReplicasToStart(host, new[] { "health-all" }, ReplicaState.Ready); + + var res = await _client.GetAsync($"http://localhost:{host.Application.Services["health-all"].Description.Bindings.First().Port}/livenessHeaders"); + Assert.True(res.IsSuccessStatusCode); + + var headers = JsonSerializer.Deserialize>(await res.Content.ReadAsStringAsync()); + Assert.Equal("value1", headers["name1"]); + Assert.Equal("value2", headers["name2"]); + + res = await _client.GetAsync($"http://localhost:{host.Application.Services["health-all"].Description.Bindings.First().Port}/readinessHeaders"); + Assert.True(res.IsSuccessStatusCode); + + headers = JsonSerializer.Deserialize>(await res.Content.ReadAsStringAsync()); + Assert.Equal("value3", headers["name3"]); + Assert.Equal("value4", headers["name4"]); + } + + private async Task SetHealthyReadyInReplica(ReplicaStatus replica, bool? healthy = null, bool? ready = null, int? healthyDelay = null, int? readyDelay = null) + { + var query = new List(); + + if (healthy.HasValue) + { + query.Add("healthy=" + healthy); + } + + if (ready.HasValue) + { + query.Add("ready=" + ready); + } + + if (healthyDelay.HasValue) + { + query.Add("healthyDelay=" + healthyDelay.Value); + } + + if (readyDelay.HasValue) + { + query.Add("readyDelay=" + readyDelay.Value); + } + + await _client.GetAsync($"http://localhost:{replica.Ports.First()}/set?" + string.Join("&", query)); + } + + private async Task ProbeNumberOfUniqueReplicas(string url) + { + // this assumes roundrobin + var unique = new HashSet(); + + string? id = null; + while (id == null || unique.Add(id)) + { + var res = await _client.GetAsync(url); + id = await res.Content.ReadAsStringAsync(); + } + + return unique.Count; + } + + private void SetReplicasInitialState(TyeHost host, bool? healthy, bool? ready, string[]? services = null) + { + if (services == null) + { + services = host.Application.Services.Select(s => s.Key).ToArray(); + } + else + { + if (services.Any(s => !host.Application.Services.ContainsKey(s))) + { + throw new ArgumentException($"not all services given in {nameof(services)} exist"); + } + } + + foreach (var service in services) + { + if (healthy.HasValue) + { + host.Application.Services[service].Description.Configuration.Add(new EnvironmentVariable("healthy") { Value = "true" }); + } + + if (ready.HasValue) + { + host.Application.Services[service].Description.Configuration.Add(new EnvironmentVariable("ready") { Value = "true" }); + } + } + } + } +} diff --git a/test/E2ETest/Microsoft.Tye.E2ETests.csproj b/test/E2ETest/Microsoft.Tye.E2ETests.csproj index 92a15968c..8eb614439 100644 --- a/test/E2ETest/Microsoft.Tye.E2ETests.csproj +++ b/test/E2ETest/Microsoft.Tye.E2ETests.csproj @@ -1,3 +1,4 @@ + @@ -30,4 +31,4 @@ - + \ No newline at end of file diff --git a/test/E2ETest/ReplicaStoppingTests.cs b/test/E2ETest/ReplicaStoppingTests.cs new file mode 100644 index 000000000..db2e56d7f --- /dev/null +++ b/test/E2ETest/ReplicaStoppingTests.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Microsoft.Tye; +using Microsoft.Tye.Hosting; +using Microsoft.Tye.Hosting.Model; +using Microsoft.Tye.Hosting.Model.V1; +using Test.Infrastructure; +using Xunit; +using Xunit.Abstractions; +using static Test.Infrastructure.TestHelpers; + +namespace E2ETest +{ + public class ReplicaStoppingTests + { + private readonly ITestOutputHelper _output; + private readonly TestOutputLogEventSink _sink; + private readonly JsonSerializerOptions _options; + + private static readonly ReplicaState?[] startedOrHigher = new ReplicaState?[] { ReplicaState.Started, ReplicaState.Healthy, ReplicaState.Ready }; + private static readonly ReplicaState?[] stoppedOrLower = new ReplicaState?[] { ReplicaState.Stopped, ReplicaState.Removed }; + + public ReplicaStoppingTests(ITestOutputHelper output) + { + _output = output; + _sink = new TestOutputLogEventSink(output); + + _options = new JsonSerializerOptions() + { + PropertyNameCaseInsensitive = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + }; + + _options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + } + + [Fact] + public async Task MultiProjectStoppingTests() + { + using var projectDirectory = CopyTestProjectDirectory("health-checks"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-none.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await RunHostingApplication(application, new HostOptions(), async (host, uri) => + { + var replicaToStop = host.Application.Services["health-none"].Replicas.First(); + Assert.Contains(replicaToStop.Value.State, startedOrHigher); + + var replicasToRestart = new[] { replicaToStop.Key }; + var restOfReplicas = host.Application.Services.SelectMany(s => s.Value.Replicas).Select(r => r.Value.Name).Where(r => r != replicaToStop.Key).ToArray(); + + Assert.True(await DoOperationAndWaitForReplicasToRestart(host, replicasToRestart.ToHashSet(), restOfReplicas.ToHashSet(), TimeSpan.FromSeconds(1), _ => + { + replicaToStop.Value.StoppingTokenSource!.Cancel(); + return Task.CompletedTask; + })); + + Assert.Contains(replicaToStop.Value.State, stoppedOrLower); + Assert.True(host.Application.Services.SelectMany(s => s.Value.Replicas).All(r => startedOrHigher.Contains(r.Value.State))); + }); + } + + [ConditionalFact] + [SkipIfDockerNotRunning] + public async Task MultiProjectDockerStoppingTests() + { + using var projectDirectory = CopyTestProjectDirectory("multi-project"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + await RunHostingApplication(application, new HostOptions() { Docker = true }, async (host, uri) => + { + var replicaToStop = host.Application.Services["frontend"].Replicas.First(); + Assert.Contains(replicaToStop.Value.State, startedOrHigher); + + var replicasToRestart = new[] { replicaToStop.Key }; + var restOfReplicas = host.Application.Services.SelectMany(s => s.Value.Replicas).Select(r => r.Value.Name).Where(r => r != replicaToStop.Key).ToArray(); + + Assert.True(await DoOperationAndWaitForReplicasToRestart(host, replicasToRestart.ToHashSet(), restOfReplicas.ToHashSet(), TimeSpan.FromSeconds(1), _ => + { + replicaToStop.Value.StoppingTokenSource!.Cancel(); + return Task.CompletedTask; + })); + + Assert.Contains(replicaToStop.Value.State, startedOrHigher); // when a container restarts, it assumes the same replica + Assert.True(host.Application.Services.SelectMany(s => s.Value.Replicas).All(r => startedOrHigher.Contains(r.Value.State))); + }); + } + + private async Task RunHostingApplication(ApplicationBuilder application, HostOptions options, Func execute) + { + await using var host = new TyeHost(application.ToHostingApplication(), options) + { + Sink = _sink, + }; + + try + { + await StartHostAndWaitForReplicasToStart(host); + + var uri = new Uri(host.DashboardWebApplication!.Addresses.First()); + + await execute(host, uri!); + } + finally + { + if (host.DashboardWebApplication != null) + { + var uri = new Uri(host.DashboardWebApplication!.Addresses.First()); + + using var client = new HttpClient(); + + foreach (var s in host.Application.Services.Values) + { + var logs = await client.GetStringAsync(new Uri(uri, $"/api/v1/logs/{s.Description.Name}")); + + _output.WriteLine($"Logs for service: {s.Description.Name}"); + _output.WriteLine(logs); + + var description = await client.GetStringAsync(new Uri(uri, $"/api/v1/services/{s.Description.Name}")); + + _output.WriteLine($"Service defintion: {s.Description.Name}"); + _output.WriteLine(description); + } + } + } + } + } +} diff --git a/test/E2ETest/TyeGenerateTests.cs b/test/E2ETest/TyeGenerateTests.cs index 6f6f1b966..4c4efb46b 100644 --- a/test/E2ETest/TyeGenerateTests.cs +++ b/test/E2ETest/TyeGenerateTests.cs @@ -392,5 +392,40 @@ public async Task Generate_Ingress() await DockerAssert.DeleteDockerImagesAsync(output, "appb"); } } + + [ConditionalFact] + [SkipIfDockerNotRunning] + public async Task Generate_HealthChecks() + { + var applicationName = "health-checks"; + var environment = "production"; + var projectName = "health-all"; + + await DockerAssert.DeleteDockerImagesAsync(output, projectName); + + using var projectDirectory = TestHelpers.CopyTestProjectDirectory(applicationName); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye-all.yaml")); + + var outputContext = new OutputContext(sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + try + { + await GenerateHost.ExecuteGenerateAsync(outputContext, application, environment, interactive: false); + + // name of application is the folder + var content = await File.ReadAllTextAsync(Path.Combine(projectDirectory.DirectoryPath, $"{applicationName}-generate-{environment}.yaml")); + var expectedContent = await File.ReadAllTextAsync($"testassets/generate/{applicationName}.yaml"); + + YamlAssert.Equals(expectedContent, content, output); + + await DockerAssert.AssertImageExistsAsync(output, projectName); + } + finally + { + await DockerAssert.DeleteDockerImagesAsync(output, projectName); + } + } } } diff --git a/test/E2ETest/testassets/generate/health-checks.yaml b/test/E2ETest/testassets/generate/health-checks.yaml new file mode 100644 index 000000000..f04fe00c6 --- /dev/null +++ b/test/E2ETest/testassets/generate/health-checks.yaml @@ -0,0 +1,76 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: health-all + labels: + app.kubernetes.io/name: 'health-all' + app.kubernetes.io/part-of: 'health-checks' +spec: + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: health-all + template: + metadata: + labels: + app.kubernetes.io/name: 'health-all' + app.kubernetes.io/part-of: 'health-checks' + spec: + containers: + - name: health-all + image: health-all:1.0.0 + imagePullPolicy: Always + env: + - name: ASPNETCORE_URLS + value: 'http://*:8004' + - name: PORT + value: '8004' + ports: + - containerPort: 8004 + livenessProbe: + httpGet: + path: /healthy + port: 8004 + scheme: HTTP + httpHeaders: + - name: name1 + value: value1 + - name: name2 + value: value2 + initialDelaySeconds: 5 + periodSeconds: 1 + successThreshold: 1 + failureThreshold: 1 + readinessProbe: + httpGet: + path: /ready + port: 8004 + scheme: HTTP + httpHeaders: + - name: name3 + value: value3 + - name: name4 + value: value4 + initialDelaySeconds: 5 + periodSeconds: 1 + successThreshold: 1 + failureThreshold: 1 +... +--- +kind: Service +apiVersion: v1 +metadata: + name: health-all + labels: + app.kubernetes.io/name: 'health-all' + app.kubernetes.io/part-of: 'health-checks' +spec: + selector: + app.kubernetes.io/name: health-all + type: ClusterIP + ports: + - name: http + protocol: TCP + port: 8004 + targetPort: 8004 +... diff --git a/test/E2ETest/testassets/projects/health-checks/api/Program.cs b/test/E2ETest/testassets/projects/health-checks/api/Program.cs new file mode 100644 index 000000000..f0c8e8ecf --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/api/Program.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace api +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(web => + { + web.UseStartup() + .ConfigureKestrel(options => {}); + }); + } +} diff --git a/test/E2ETest/testassets/projects/health-checks/api/Properties/launchSettings.json b/test/E2ETest/testassets/projects/health-checks/api/Properties/launchSettings.json new file mode 100644 index 000000000..d16c49475 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/api/Properties/launchSettings.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json.schemastore.org/launchsettings.json", + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:39065", + "sslPort": 44337 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "launchUrl": "weatherforecast", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "api": { + "commandName": "Project", + "launchBrowser": true, + "launchUrl": "weatherforecast", + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/test/E2ETest/testassets/projects/health-checks/api/Startup.cs b/test/E2ETest/testassets/projects/health-checks/api/Startup.cs new file mode 100644 index 000000000..139f456fd --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/api/Startup.cs @@ -0,0 +1,166 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.HttpsPolicy; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace api +{ + public class Startup + { + private static string _randomId = Guid.NewGuid().ToString(); + + private static bool _healthy = false; + private static bool _ready = false; + + private static int _healthyDelay = 0; + private static int _readyDelay = 0; + + private static Dictionary _livenessHeaders; + private static Dictionary _readinessHeaders; + + private static int[] _ports; + + private static object _locker = new object(); + + public Startup(IConfiguration configuration) + { + Configuration = configuration; + + var healthyEnv = Environment.GetEnvironmentVariable("healthy"); + var readyEnv = Environment.GetEnvironmentVariable("ready"); + + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("PORT"))) + { + var portParts = Environment.GetEnvironmentVariable("PORT").Split(';'); + _ports = portParts.Select(p => int.Parse(p)).ToArray(); + } + + if (healthyEnv != null) + { + _healthy = bool.Parse(healthyEnv); + } + + if (readyEnv != null) + { + _ready = bool.Parse(readyEnv); + } + } + + public IConfiguration Configuration { get; } + + // This method gets called by the runtime. Use this method to add services to the container. + public void ConfigureServices(IServiceCollection services) + { + services.AddControllers(); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseAuthorization(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGet("/", async ctx => + { + await ctx.Response.WriteAsync("Hello"); + }); + + endpoints.MapGet("/ports", async ctx => + { + await ctx.Response.WriteAsync(JsonSerializer.Serialize(_ports)); + }); + + endpoints.MapGet("/id", async ctx => + { + await ctx.Response.WriteAsync(_randomId); + }); + + endpoints.MapGet("/healthy", async ctx => + { + if (_healthyDelay != 0) + { + await Task.Delay(TimeSpan.FromSeconds(_healthyDelay)); + } + + _livenessHeaders = ctx.Request.Headers.ToDictionary(h => h.Key, h => h.Value.ToString()); + + ctx.Response.StatusCode = _healthy ? 200 : 500; + await ctx.Response.WriteAsync(ctx.Response.StatusCode.ToString()); + }); + + endpoints.MapGet("/ready", async ctx => + { + if (_readyDelay != 0) + { + await Task.Delay(TimeSpan.FromSeconds(_readyDelay)); + } + + _readinessHeaders = ctx.Request.Headers.ToDictionary(h => h.Key, h => h.Value.ToString()); + + ctx.Response.StatusCode = _ready ? 200 : 500; + await ctx.Response.WriteAsync(ctx.Response.StatusCode.ToString()); + }); + + // Should be technically POST/PUT, but it's just for tests... + endpoints.MapGet("/set", async ctx => + { + var query = ctx.Request.Query.ToDictionary(kv => kv.Key).ToDictionary(kv => kv.Key, kv => kv.Value.Value.First()); + if (query.ContainsKey("healthy")) + { + _healthy = bool.Parse(query["healthy"]); + } + + if (query.ContainsKey("ready")) + { + _ready = bool.Parse(query["ready"]); + } + + if (query.ContainsKey("healthyDelay")) + { + _healthyDelay = int.Parse(query["healthyDelay"]); + } + + if (query.ContainsKey("readyDelay")) + { + _readyDelay = int.Parse(query["readyDelay"]); + } + + await ctx.Response.WriteAsync(_randomId); + }); + + endpoints.MapGet("/livenessHeaders", async ctx => + { + await ctx.Response.WriteAsync(JsonSerializer.Serialize(_livenessHeaders)); + }); + + endpoints.MapGet("/readinessHeaders", async ctx => + { + await ctx.Response.WriteAsync(JsonSerializer.Serialize(_readinessHeaders)); + }); + }); + } + } +} diff --git a/test/E2ETest/testassets/projects/health-checks/api/api.csproj b/test/E2ETest/testassets/projects/health-checks/api/api.csproj new file mode 100644 index 000000000..d12c450b7 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/api/api.csproj @@ -0,0 +1,8 @@ + + + + netcoreapp3.1 + + + + diff --git a/test/E2ETest/testassets/projects/health-checks/api/appsettings.Development.json b/test/E2ETest/testassets/projects/health-checks/api/appsettings.Development.json new file mode 100644 index 000000000..8983e0fc1 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/api/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/test/E2ETest/testassets/projects/health-checks/api/appsettings.json b/test/E2ETest/testassets/projects/health-checks/api/appsettings.json new file mode 100644 index 000000000..d9d9a9bff --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/api/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/test/E2ETest/testassets/projects/health-checks/health-checks.sln b/test/E2ETest/testassets/projects/health-checks/health-checks.sln new file mode 100644 index 000000000..21cb80ccd --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/health-checks.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "api", "api\api.csproj", "{239E0475-3113-4C1C-A517-38AD9188B65D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x64.ActiveCfg = Debug|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x64.Build.0 = Debug|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x86.ActiveCfg = Debug|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Debug|x86.Build.0 = Debug|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Release|Any CPU.Build.0 = Release|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x64.ActiveCfg = Release|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x64.Build.0 = Release|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x86.ActiveCfg = Release|Any CPU + {239E0475-3113-4C1C-A517-38AD9188B65D}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/test/E2ETest/testassets/projects/health-checks/tye-all.yaml b/test/E2ETest/testassets/projects/health-checks/tye-all.yaml new file mode 100644 index 000000000..b88c00646 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/tye-all.yaml @@ -0,0 +1,36 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +name: health-checks +services: + - name: health-all + project: api/api.csproj + replicas: 3 + bindings: + - port: 8004 + liveness: + http: + path: /healthy + headers: + - name: name1 + value: value1 + - name: name2 + value: value2 + initialDelay: 5 + period: 1 + timeout: 1 + successThreshold: 1 + failureThreshold: 1 + readiness: + http: + path: /ready + headers: + - name: name3 + value: value3 + - name: name4 + value: value4 + initialDelay: 5 + period: 1 + timeout: 2 + successThreshold: 1 + failureThreshold: 1 + \ No newline at end of file diff --git a/test/E2ETest/testassets/projects/health-checks/tye-ingress.yaml b/test/E2ETest/testassets/projects/health-checks/tye-ingress.yaml new file mode 100644 index 000000000..3f39b9070 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/tye-ingress.yaml @@ -0,0 +1,30 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +name: health-checks +ingress: + - name: ingress + bindings: + - port: 8006 + rules: + - path: /api + service: health-ingress-svc +services: + - name: health-ingress-svc + project: api/api.csproj + replicas: 2 + liveness: + http: + path: /healthy + initialDelay: 5 + period: 1 + timeout: 1 + successThreshold: 1 + failureThreshold: 1 + readiness: + http: + path: /ready + initialDelay: 5 + period: 1 + timeout: 1 + successThreshold: 1 + failureThreshold: 1 diff --git a/test/E2ETest/testassets/projects/health-checks/tye-liveness.yaml b/test/E2ETest/testassets/projects/health-checks/tye-liveness.yaml new file mode 100644 index 000000000..0a4d43a93 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/tye-liveness.yaml @@ -0,0 +1,17 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +name: health-checks +services: + - name: health-liveness + project: api/api.csproj + replicas: 3 + bindings: + - port: 8002 + liveness: + http: + path: /healthy + initialDelay: 5 + period: 1 + timeout: 1 + successThreshold: 1 + failureThreshold: 1 \ No newline at end of file diff --git a/test/E2ETest/testassets/projects/health-checks/tye-none.yaml b/test/E2ETest/testassets/projects/health-checks/tye-none.yaml new file mode 100644 index 000000000..44cb5f537 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/tye-none.yaml @@ -0,0 +1,10 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +name: health-checks +services: + - name: health-none + project: api/api.csproj + replicas: 3 + bindings: + - port: 8001 + \ No newline at end of file diff --git a/test/E2ETest/testassets/projects/health-checks/tye-proxy.yaml b/test/E2ETest/testassets/projects/health-checks/tye-proxy.yaml new file mode 100644 index 000000000..3469fb498 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/tye-proxy.yaml @@ -0,0 +1,26 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +name: health-checks +services: + - name: health-proxy + project: api/api.csproj + replicas: 2 + bindings: + - port: 8005 + liveness: + http: + path: /healthy + initialDelay: 5 + period: 1 + timeout: 1 + successThreshold: 1 + failureThreshold: 1 + readiness: + http: + path: /ready + initialDelay: 5 + period: 1 + timeout: 1 + successThreshold: 1 + failureThreshold: 1 + \ No newline at end of file diff --git a/test/E2ETest/testassets/projects/health-checks/tye-readiness.yaml b/test/E2ETest/testassets/projects/health-checks/tye-readiness.yaml new file mode 100644 index 000000000..abe7fae02 --- /dev/null +++ b/test/E2ETest/testassets/projects/health-checks/tye-readiness.yaml @@ -0,0 +1,17 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +name: health-checks +services: + - name: health-readiness + project: api/api.csproj + replicas: 3 + bindings: + - port: 8003 + readiness: + http: + path: /ready + initialDelay: 5 + period: 1 + timeout: 1 + successThreshold: 1 + failureThreshold: 1 \ No newline at end of file diff --git a/test/Test.Infrastructure/TestHelpers.cs b/test/Test.Infrastructure/TestHelpers.cs index bbc71ef28..6286df927 100644 --- a/test/Test.Infrastructure/TestHelpers.cs +++ b/test/Test.Infrastructure/TestHelpers.cs @@ -101,38 +101,50 @@ public static TempDirectory CopyTestProjectDirectory(string projectName) return temp; } - public static async Task StartHostAndWaitForReplicasToStart(TyeHost host) + public static async Task DoOperationAndWaitForReplicasToChangeState(TyeHost host, ReplicaState desiredState, int n, HashSet? toChange, HashSet? rest, Func entitySelector, TimeSpan waitUntilSuccess, Func operation) { - var startedTask = new TaskCompletionSource(); - var alreadyStarted = 0; - var totalReplicas = host.Application.Services.Sum(s => s.Value.Description.Replicas); + if (toChange != null && rest != null && rest.Overlaps(toChange)) + { + throw new ArgumentException($"{nameof(toChange)} and {nameof(rest)} can't overlap"); + } + + var changedTask = new TaskCompletionSource(); + var remaining = n; void OnReplicaChange(ReplicaEvent ev) { - if (ev.State == ReplicaState.Started) + if (rest != null && rest.Contains(entitySelector(ev))) { - Interlocked.Increment(ref alreadyStarted); + changedTask!.TrySetResult(false); } - else if (ev.State == ReplicaState.Stopped) + else if ((toChange == null || toChange.Contains(entitySelector(ev))) && ev.State == desiredState) { - Interlocked.Decrement(ref alreadyStarted); + Interlocked.Decrement(ref remaining); } - if (alreadyStarted == totalReplicas) + if (remaining == 0) { - startedTask!.TrySetResult(true); + Task.Delay(waitUntilSuccess) + .ContinueWith(_ => + { + if (!changedTask!.Task.IsCompleted) + { + changedTask!.TrySetResult(remaining == 0); + } + }); } } var servicesStateObserver = host.Application.Services.Select(srv => srv.Value.ReplicaEvents.Subscribe(OnReplicaChange)).ToList(); - await host.StartAsync(); + + await operation(host); using var cancellation = new CancellationTokenSource(WaitForServicesTimeout); try { - await using (cancellation.Token.Register(() => startedTask.TrySetCanceled())) + await using (cancellation.Token.Register(() => changedTask.TrySetCanceled())) { - await startedTask.Task; + return await changedTask.Task; } } finally @@ -144,46 +156,58 @@ void OnReplicaChange(ReplicaEvent ev) } } - public static async Task PurgeHostAndWaitForGivenReplicasToStop(TyeHost host, string[] replicas) + public static async Task DoOperationAndWaitForReplicasToRestart(TyeHost host, HashSet toRestart, HashSet? rest, Func entitySelector, TimeSpan waitUntilSuccess, Func operation) { - static async Task Purge(TyeHost host) + if (rest != null && rest.Overlaps(toRestart)) { - var logger = host.DashboardWebApplication!.Logger; - var replicaRegistry = new ReplicaRegistry(host.Application.ContextDirectory, logger); - var processRunner = new ProcessRunner(logger, replicaRegistry, new ProcessRunnerOptions()); - var dockerRunner = new DockerRunner(logger, replicaRegistry); - - await processRunner.StartAsync(new Application(new FileInfo(host.Application.Source), new Dictionary())); - await dockerRunner.StartAsync(new Application(new FileInfo(host.Application.Source), new Dictionary())); + throw new ArgumentException($"{nameof(toRestart)} and {nameof(rest)} can't overlap"); } - var stoppedTask = new TaskCompletionSource(); - var remaining = replicas.Length; + var restartedTask = new TaskCompletionSource(); + var remaining = toRestart.Count; + var alreadyStarted = 0; void OnReplicaChange(ReplicaEvent ev) { - if (replicas.Contains(ev.Replica.Name) && ev.State == ReplicaState.Stopped) + if (ev.State == ReplicaState.Started) { - Interlocked.Decrement(ref remaining); + Interlocked.Increment(ref alreadyStarted); + } + else if (ev.State == ReplicaState.Stopped) + { + if (toRestart.Contains(entitySelector(ev))) + { + Interlocked.Decrement(ref remaining); + } + else if (rest != null && rest.Contains(entitySelector(ev))) + { + restartedTask!.SetResult(false); + } } - if (remaining == 0) + if (remaining == 0 && alreadyStarted == toRestart.Count) { - stoppedTask!.TrySetResult(true); + Task.Delay(waitUntilSuccess) + .ContinueWith(_ => + { + if (!restartedTask!.Task.IsCompleted) + { + restartedTask!.SetResult(remaining == 0 && alreadyStarted == toRestart.Count); + } + }); } } var servicesStateObserver = host.Application.Services.Select(srv => srv.Value.ReplicaEvents.Subscribe(OnReplicaChange)).ToList(); - // We purge existing replicas by restarting the host which will initiate the purging process - await Purge(host); + await operation(host); using var cancellation = new CancellationTokenSource(WaitForServicesTimeout); try { - await using (cancellation.Token.Register(() => stoppedTask.TrySetCanceled())) + await using (cancellation.Token.Register(() => restartedTask.TrySetCanceled())) { - await stoppedTask.Task; + return await restartedTask.Task; } } finally @@ -194,5 +218,44 @@ void OnReplicaChange(ReplicaEvent ev) } } } + + public static Task DoOperationAndWaitForReplicasToChangeState(TyeHost host, ReplicaState desiredState, int n, HashSet? toChange, HashSet? rest, TimeSpan waitUntilSuccess, Func operation) + => DoOperationAndWaitForReplicasToChangeState(host, desiredState, n, toChange, rest, ev => ev.Replica.Name, waitUntilSuccess, operation); + + public static Task DoOperationAndWaitForReplicasToRestart(TyeHost host, HashSet toRestart, HashSet? rest, TimeSpan waitUntilSuccess, Func operation) + => DoOperationAndWaitForReplicasToRestart(host, toRestart, rest, ev => ev.Replica.Name, waitUntilSuccess, operation); + + public static async Task StartHostAndWaitForReplicasToStart(TyeHost host, string[]? services = null, ReplicaState desiredState = ReplicaState.Started) + { + if (services == null) + { + await DoOperationAndWaitForReplicasToChangeState(host, desiredState, host.Application.Services.Sum(s => s.Value.Description.Replicas), null, null, TimeSpan.Zero, h => h.StartAsync()); + } + else + { + if (services.Any(s => !host.Application.Services.ContainsKey(s))) + { + throw new ArgumentException($"not all services given in {nameof(services)} exist"); + } + + await DoOperationAndWaitForReplicasToChangeState(host, desiredState, host.Application.Services.Where(s => services.Contains(s.Value.Description.Name)).Sum(s => s.Value.Description.Replicas), services.ToHashSet(), null, ev => ev.Replica.Service.Description.Name, TimeSpan.Zero, h => h.StartAsync()); + } + } + + public static async Task PurgeHostAndWaitForGivenReplicasToStop(TyeHost host, string[] replicas) + { + static async Task Purge(TyeHost host) + { + var logger = host.DashboardWebApplication!.Logger; + var replicaRegistry = new ReplicaRegistry(host.Application.ContextDirectory, logger); + var processRunner = new ProcessRunner(logger, replicaRegistry, new ProcessRunnerOptions()); + var dockerRunner = new DockerRunner(logger, replicaRegistry); + + await processRunner.StartAsync(new Application(new FileInfo(host.Application.Source), new Dictionary())); + await dockerRunner.StartAsync(new Application(new FileInfo(host.Application.Source), new Dictionary())); + } + + await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Stopped, replicas.Length, replicas.ToHashSet(), new HashSet(), TimeSpan.Zero, Purge); + } } }