From aff652e0f601013c10c32d0e04a0a94fe2f55080 Mon Sep 17 00:00:00 2001 From: Ryan Nowak Date: Sun, 3 May 2020 15:36:27 -0700 Subject: [PATCH] Add recipe for Elastic stack (ELK) --- docs/recipes/logging.md | 101 ++++++++++++++++++ .../Elastic/ElasticStackExtension.cs | 87 +++++++++++++++ .../WellKnownExtensions.cs | 2 + .../LoggingSink.cs | 2 +- 4 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 docs/recipes/logging.md create mode 100644 src/Microsoft.Tye.Extensions/Elastic/ElasticStackExtension.cs diff --git a/docs/recipes/logging.md b/docs/recipes/logging.md new file mode 100644 index 000000000..7739e7f46 --- /dev/null +++ b/docs/recipes/logging.md @@ -0,0 +1,101 @@ +# Logging with Elastic stack + +> :warning: This recipe refers to features that are only available in our CI builds at the moment. These features will be part of the 0.2 release on nuget.org "soon". + +Elastic stack (aka ELK) is a popular log-aggregation system that gives you a powerful log search and dashboard engine with views across all of your services. + +Tye can push logs to Elastic stack easily without the need for any SDKs or code changes in your services. + +## Getting started: running locally with Elastic stack + +> :bulb: If you want an existing sample to run, the [sample here](https://github.com/dotnet/tye/tree/master/samples/frontend-backend) will do. This recipe will show examples of UI and data based on that sample. You own application will work fine, but the data and examples will look different. + +The first step is to add the `elastic` extension to your `tye.yaml`. Add the `extensions` node and its children from the example below. + +```yaml +name: frontend-backend + +extensions: +- name: elastic + logPath: ./.logs + +services: +- name: backend + project: backend/backend.csproj +- name: frontend + project: frontend/frontend.csproj +``` + +The `logPath` property here configures the path where elasticsearch will store its data. + +> :bulb: Tye can successfully launch Elastic stack without `logPath`, but ... It's *highly* recommended that you specify a path to store the logs and configuration (add to `.gitignore` if it's part of your repository). Kibana has some mandatory setup the first time you use it, and without persisting the data, you will have to go through it each time. + +Now launch the application with Tye: + +```sh +tye run +``` + +If you navigate to the Tye dashboard you should see an extra service (`elastic`) in the list of services. + +image + +Elastic stack can take 30s to a few minutes to start. Visit the first URI (`http://localhost:5601`) in your browser to access the Kibana dashboard. This make some patience (you can watch the logs from the dashboard). + +Once is up you should see a welcome screen asking you to do some configuration: + +image + +To configure Kibana click on the link near the bottom right of this screenshot: `Connect to your Elasticsearch Index`. + +The next step will ask you to configure the index, enter `logstash-*` in the textbox and shown below: + +image + +And click on `> Next step` to advance. + +The last configuration step will ask you to choose the field from your log data that represents times. Choose `@timestamp`. + +image + +Click `Create index pattern` to save. It should look roughly like the screenshot below when complete: + +image + +Now you're ready to view the data! Click the `Discover` icon. That's the icon on the toolbar on the left at the top. It looks like a compass. + +After that loads, it should look like the screenshot below: + +image + +Now you can see the logs from your application with each field broken out into structured format. If you take advantage of [structured logging](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/logging/?view=aspnetcore-3.1#log-message-template) then you'll see your own data included in structured form here alongside framework logs. + +## Understanding log aggregation and tracing + +Now that we've got a logging dashboard, let's do a query to understand the connection between tracing and log aggregation. + +The basic idea is that because log aggregation pulls all of the logs together across all of your services, and distributed tracing flows context across services, you can easily query all of the logs for a logical operation. + +We'll use the discover tab to build a basic query. + +First, perform some operation that will trigger a cross-service call so we have some data to use. If you're using the [sample here](https://github.com/dotnet/tye/tree/master/samples/frontend-backend) then visiting the frontend application in the browser will do. + +Next, let's add some fields to the query. The left pane in Kibana has all of the known fields. Select the following by clicking on the `Add` button near each of them: + +- `message` +- `fields.TraceId` +- `fields.Application` + +It should like the screenshot below: + +image + +You can expand an individual message by clicking the `>` icon on the left if you want to see the values of non-selected fields. + +Now it's showing just the fields that were selected, but its showing all log messages, not the ones from a specific request. + +Hover over on of the values of `fields.TraceId` for a request to `backend` and click the icon for `Filter for value` (magnifying glass with the `+`). + +image + +Now you can see all of the log messages for just that operation, across both the `frontend` and `backend` services. This kind of query can be useful when you want to investigate a problem that occured in the past, and see exactly what was logged by each service that participated in the operation. \ No newline at end of file diff --git a/src/Microsoft.Tye.Extensions/Elastic/ElasticStackExtension.cs b/src/Microsoft.Tye.Extensions/Elastic/ElasticStackExtension.cs new file mode 100644 index 000000000..740f6dc5f --- /dev/null +++ b/src/Microsoft.Tye.Extensions/Elastic/ElasticStackExtension.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Tye.Extensions.Elastic +{ + internal sealed class ElasticStackExtension : Extension + { + public override Task ProcessAsync(ExtensionContext context, ExtensionConfiguration config) + { + if (context.Application.Services.Any(s => s.Name == "elastic")) + { + context.Output.WriteDebugLine("elastic service already configured. Skipping..."); + } + else + { + context.Output.WriteDebugLine("Injecting elastic service..."); + + // We're using an "all-in-one" docker image for local dev that makes that + // easy to set up. + // + // See: https://elk-docker.readthedocs.io/ + var elastic = new ContainerServiceBuilder("elastic", "sebp/elk") + { + Bindings = + { + new BindingBuilder() + { + Name = "kibbana", + Port = 5601, + ContainerPort = 5601, + Protocol = "http", + }, + new BindingBuilder() + { + Name = "elastic", + Port = 9200, + ContainerPort = 9200, + Protocol = "http", + }, + }, + }; + context.Application.Services.Add(elastic); + + if (config.Data.TryGetValue("logPath", out var obj) && + obj is string logPath && + !string.IsNullOrEmpty(logPath)) + { + // https://elk-docker.readthedocs.io/#persisting-log-data + elastic.Volumes.Add(new VolumeBuilder(logPath, "elk-data", "/var/lib/elasticsearch")); + } + + foreach (var s in context.Application.Services) + { + if (object.ReferenceEquals(s, elastic)) + { + continue; + } + + // make elastic available as a dependency of everything. + if (!s.Dependencies.Contains(elastic.Name)) + { + s.Dependencies.Add(elastic.Name); + } + } + } + + if (context.Operation == ExtensionContext.OperationKind.LocalRun) + { + if (context.Options!.LoggingProvider is null) + { + // For local development we hardcode the port and hostname + context.Options.LoggingProvider = "elastic=http://localhost:9200"; + } + } + else if (context.Operation == ExtensionContext.OperationKind.Deploy) + { + // TODO: support this :) + } + + return Task.CompletedTask; + } + } +} diff --git a/src/Microsoft.Tye.Extensions/WellKnownExtensions.cs b/src/Microsoft.Tye.Extensions/WellKnownExtensions.cs index 401982122..4457b179f 100644 --- a/src/Microsoft.Tye.Extensions/WellKnownExtensions.cs +++ b/src/Microsoft.Tye.Extensions/WellKnownExtensions.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using Microsoft.Tye.Extensions.Dapr; +using Microsoft.Tye.Extensions.Elastic; using Microsoft.Tye.Extensions.Zipkin; namespace Microsoft.Tye.Extensions @@ -14,6 +15,7 @@ public static class WellKnownExtensions public static IReadOnlyDictionary Extensions = new Dictionary(StringComparer.InvariantCultureIgnoreCase) { { "dapr", new DaprExtension() }, + { "elastic", new ElasticStackExtension() }, { "zipkin", new ZipkinExtension() }, }; } diff --git a/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs b/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs index b14ee5f6a..394e47e1b 100644 --- a/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs +++ b/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs @@ -26,7 +26,7 @@ public LoggingSink(Microsoft.Extensions.Logging.ILogger logger, DiagnosticsProvi public IDisposable Attach(EventPipeEventSource source, ReplicaInfo replicaInfo) { - using var loggerFactory = LoggerFactory.Create(builder => ConfigureLogging(replicaInfo.Service, replicaInfo.Replica, builder)); + var loggerFactory = LoggerFactory.Create(builder => ConfigureLogging(replicaInfo.Service, replicaInfo.Replica, builder)); var lastFormattedMessage = "";