From 40bcd6de4427646426ae70080bf01265584da0d4 Mon Sep 17 00:00:00 2001 From: Thomas Pierce Date: Wed, 29 Mar 2023 16:28:34 -0700 Subject: [PATCH] Add new components to allow for generating metrics from 100% of spans without impacting sampling In this commit, we are adding several new components to the awsxray package: * AlwaysRecordSampler - A simple aggregate sampler that always ensures spans are "recorded" (i.e. sent to span processors). This allows us to generate metrics from 100% of spans without impacting true sampling rate. * SpanMetricsProcessor - A span processor that will generate specific metrics pertaining to latency, faults, and errors. Relies on a MetricAttributeGenerator to build attributes for the metrics, and wraps these metric attributes around the span attributes before passing the span to a delegate span processor for further processing/exporting. * MetricAttributeGenerator - A generic interface for components that consume spans and resources and export attributes for metrics generated from these spans. * AwsMetricAttributeGenerator - A specific implementation of MetricAttributeGenerator, used for generating AWS-specific attributes. * SpanMetricsProcessorBuilder - A builder class for SpanMetricsProcessor Related issue: https://github.com/open-telemetry/opentelemetry-java-contrib/issues/789 --- .../contrib/awsxray/AlwaysRecordSampler.java | 84 ++++ .../awsxray/AwsMetricAttributeGenerator.java | 229 ++++++++++ .../awsxray/MetricAttributeGenerator.java | 28 ++ .../contrib/awsxray/SpanMetricsProcessor.java | 278 ++++++++++++ .../awsxray/SpanMetricsProcessorBuilder.java | 85 ++++ .../awsxray/AlwaysRecordSamplerTest.java | 108 +++++ .../AwsMetricAttributeGeneratorTest.java | 321 ++++++++++++++ .../awsxray/SpanMetricsProcessorTest.java | 409 ++++++++++++++++++ 8 files changed, 1542 insertions(+) create mode 100644 aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSampler.java create mode 100644 aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGenerator.java create mode 100644 aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/MetricAttributeGenerator.java create mode 100644 aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessor.java create mode 100644 aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorBuilder.java create mode 100644 aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSamplerTest.java create mode 100644 aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGeneratorTest.java create mode 100644 aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorTest.java diff --git a/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSampler.java b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSampler.java new file mode 100644 index 000000000..c2d65f020 --- /dev/null +++ b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSampler.java @@ -0,0 +1,84 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.List; +import javax.annotation.concurrent.Immutable; + +/** + * This sampler will return the sampling result of the provided {@link #rootSampler}, unless the + * sampling result contains the sampling decision {@link SamplingDecision#DROP}, in which case, a + * new sampling result will be returned that is functionally equivalent to the original, except that + * it contains the sampling decision {@link SamplingDecision#RECORD_ONLY}. This ensures that all + * spans are recorded, with no change to sampling. + * + *

The intended use case of this sampler is to provide a means of sending all spans to a + * processor without having an impact on the sampling rate. This may be desirable if a user wishes + * to count or otherwise measure all spans produced in an application, without incurring the cost of + * 100% sampling. + */ +@Immutable +public final class AlwaysRecordSampler implements Sampler { + + private final Sampler rootSampler; + + public static AlwaysRecordSampler create(Sampler rootSampler) { + return new AlwaysRecordSampler(rootSampler); + } + + private AlwaysRecordSampler(Sampler rootSampler) { + this.rootSampler = rootSampler; + } + + @Override + public SamplingResult shouldSample( + Context parentContext, + String traceId, + String name, + SpanKind spanKind, + Attributes attributes, + List parentLinks) { + SamplingResult result = + rootSampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + if (result.getDecision() == SamplingDecision.DROP) { + result = wrapResultWithRecordOnlyResult(result); + } + + return result; + } + + @Override + public String getDescription() { + return "AlwaysRecordSampler{" + rootSampler.getDescription() + "}"; + } + + private static SamplingResult wrapResultWithRecordOnlyResult(SamplingResult result) { + return new SamplingResult() { + @Override + public SamplingDecision getDecision() { + return SamplingDecision.RECORD_ONLY; + } + + @Override + public Attributes getAttributes() { + return result.getAttributes(); + } + + @Override + public TraceState getUpdatedTraceState(TraceState parentTraceState) { + return result.getUpdatedTraceState(parentTraceState); + } + }; + } +} diff --git a/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGenerator.java b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGenerator.java new file mode 100644 index 000000000..f4b4769a5 --- /dev/null +++ b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGenerator.java @@ -0,0 +1,229 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import static io.opentelemetry.contrib.awsxray.AwsAttributeKeys.AWS_LOCAL_OPERATION; +import static io.opentelemetry.contrib.awsxray.AwsAttributeKeys.AWS_REMOTE_APPLICATION; +import static io.opentelemetry.contrib.awsxray.AwsAttributeKeys.AWS_REMOTE_OPERATION; +import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.DB_OPERATION; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.DB_SYSTEM; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.FAAS_INVOKED_NAME; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.FAAS_INVOKED_PROVIDER; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.GRAPHQL_OPERATION_TYPE; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.MESSAGING_OPERATION; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.MESSAGING_SYSTEM; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.PEER_SERVICE; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.RPC_METHOD; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.RPC_SERVICE; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import io.opentelemetry.semconv.trace.attributes.SemanticAttributes; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * AwsMetricAttributeGenerator generates very specific metric attributes based on low-cardinality + * span and resource attributes. If such attributes are not present, we fallback to default values. + * + *

The goal of these particular metric attributes is to get metrics for incoming and outgoing + * traffic for an application. Namely, {@link SpanKind#SERVER} amd {@link SpanKind#CONSUMER} spans + * represent "incoming" traffic, {@link SpanKind#CLIENT} and {@link SpanKind#PRODUCER} spans + * represent "outgoing" traffic, and {@link SpanKind#INTERNAL} spans are ignored. + */ +final class AwsMetricAttributeGenerator implements MetricAttributeGenerator { + + private static final Logger logger = + Logger.getLogger(AwsMetricAttributeGenerator.class.getName()); + + // Generated metric attribute keys + private static final AttributeKey APPLICATION = AttributeKey.stringKey("Application"); + private static final AttributeKey OPERATION = AttributeKey.stringKey("Operation"); + private static final AttributeKey REMOTE_APPLICATION = + AttributeKey.stringKey("RemoteApplication"); + private static final AttributeKey REMOTE_OPERATION = + AttributeKey.stringKey("RemoteOperation"); + private static final AttributeKey SPAN_KIND = AttributeKey.stringKey("span.kind"); + + // Special APPLICATION attribute value if GRAPHQL_OPERATION_TYPE attribute key is present. + private static final String GRAPHQL = "graphql"; + + // Default attribute values if no valid span attribute value is identified + private static final String UNKNOWN_APPLICATION = "UnknownApplication"; + private static final String UNKNOWN_OPERATION = "UnknownOperation"; + private static final String UNKNOWN_REMOTE_APPLICATION = "UnknownRemoteApplication"; + private static final String UNKNOWN_REMOTE_OPERATION = "UnknownRemoteOperation"; + + @Override + public Attributes generateMetricAttributesFromSpan(ReadableSpan span, Resource resource) { + AttributesBuilder builder = Attributes.builder(); + switch (span.getKind()) { + case CONSUMER: + case SERVER: + setApplication(resource, span, builder); + setIngressOperation(span, builder); + setSpanKind(span, builder); + break; + case PRODUCER: + case CLIENT: + setApplication(resource, span, builder); + setEgressOperation(span, builder); + setRemoteApplicationAndOperation(span, builder); + setSpanKind(span, builder); + break; + default: + // Add no attributes, signalling no metrics should be emitted. + } + return builder.build(); + } + + /** Application is always derived from {@link ResourceAttributes#SERVICE_NAME} */ + private static void setApplication( + Resource resource, ReadableSpan span, AttributesBuilder builder) { + String application = resource.getAttribute(SERVICE_NAME); + if (application == null) { + logUnknownAttribute(APPLICATION, span); + application = UNKNOWN_APPLICATION; + } + builder.put(APPLICATION, application); + } + + /** + * Ingress operation (i.e. operation for Server and Consumer spans) is always derived from span + * name. + */ + private static void setIngressOperation(ReadableSpan span, AttributesBuilder builder) { + String operation = span.getName(); + if (operation == null) { + logUnknownAttribute(OPERATION, span); + operation = UNKNOWN_OPERATION; + } + builder.put(OPERATION, operation); + } + + /** + * Egress operation (i.e. operation for Client and Producer spans) is always derived from a + * special span attribute, {@link AwsAttributeKeys#AWS_LOCAL_OPERATION}. This attribute is + * generated with a separate SpanProcessor, {@link LocalAttributesSpanProcessor} + */ + private static void setEgressOperation(ReadableSpan span, AttributesBuilder builder) { + String operation = span.getAttribute(AWS_LOCAL_OPERATION); + if (operation == null) { + logUnknownAttribute(OPERATION, span); + operation = UNKNOWN_OPERATION; + } + builder.put(OPERATION, operation); + } + + /** + * Remote attributes (only for Client and Producer spans) are generated based on low-cardinality + * span attributes, in priority order. + * + *

The first priority is the AWS Remote attributes, which are generated from manually + * instrumented span attributes, and are clear indications of customer intent. If AWS Remote + * attributes are not present, the next highest priority span attribute is Peer Service, which is + * also a reliable indicator of customer intent. If this is set, it will override {@link + * #REMOTE_APPLICATION} identified from any other span attribute, other than AWS Remote + * attributes. + * + *

After this, we look for the following low-c∂ardinality span attributes that can be used to + * determine the remote metric attributes: + * + *

+ * + *

In each case, these span attributes were selected from the OpenTelemetry trace semantic + * convention specifications as they adhere to the three following criteria: + * + *

+ */ + private static void setRemoteApplicationAndOperation( + ReadableSpan span, AttributesBuilder builder) { + if (isKeyPresent(span, AWS_REMOTE_APPLICATION) || isKeyPresent(span, AWS_REMOTE_OPERATION)) { + setRemoteApplication(span, builder, AWS_REMOTE_APPLICATION); + setRemoteOperation(span, builder, AWS_REMOTE_OPERATION); + } else if (isKeyPresent(span, RPC_SERVICE) || isKeyPresent(span, RPC_METHOD)) { + setRemoteApplication(span, builder, RPC_SERVICE); + setRemoteOperation(span, builder, RPC_METHOD); + } else if (isKeyPresent(span, DB_SYSTEM) || isKeyPresent(span, DB_OPERATION)) { + setRemoteApplication(span, builder, DB_SYSTEM); + setRemoteOperation(span, builder, DB_OPERATION); + } else if (isKeyPresent(span, FAAS_INVOKED_PROVIDER) || isKeyPresent(span, FAAS_INVOKED_NAME)) { + setRemoteApplication(span, builder, FAAS_INVOKED_PROVIDER); + setRemoteOperation(span, builder, FAAS_INVOKED_NAME); + } else if (isKeyPresent(span, MESSAGING_SYSTEM) || isKeyPresent(span, MESSAGING_OPERATION)) { + setRemoteApplication(span, builder, MESSAGING_SYSTEM); + setRemoteOperation(span, builder, MESSAGING_OPERATION); + } else if (isKeyPresent(span, GRAPHQL_OPERATION_TYPE)) { + builder.put(REMOTE_APPLICATION, GRAPHQL); + setRemoteOperation(span, builder, GRAPHQL_OPERATION_TYPE); + } else { + logUnknownAttribute(REMOTE_APPLICATION, span); + builder.put(REMOTE_APPLICATION, UNKNOWN_REMOTE_APPLICATION); + logUnknownAttribute(REMOTE_OPERATION, span); + builder.put(REMOTE_OPERATION, UNKNOWN_REMOTE_OPERATION); + } + + // Peer service takes priority as RemoteApplication over everything but AWS Remote. + if (isKeyPresent(span, PEER_SERVICE) && !isKeyPresent(span, AWS_REMOTE_APPLICATION)) { + setRemoteApplication(span, builder, PEER_SERVICE); + } + } + + /** Span kind is needed for differentiating metrics in the EMF exporter */ + private static void setSpanKind(ReadableSpan span, AttributesBuilder builder) { + String spanKind = span.getKind().name(); + builder.put(SPAN_KIND, spanKind); + } + + private static boolean isKeyPresent(ReadableSpan span, AttributeKey key) { + return span.getAttribute(key) != null; + } + + private static void setRemoteApplication( + ReadableSpan span, AttributesBuilder builder, AttributeKey remoteApplicationKey) { + String remoteApplication = span.getAttribute(remoteApplicationKey); + if (remoteApplication == null) { + logUnknownAttribute(REMOTE_APPLICATION, span); + remoteApplication = UNKNOWN_REMOTE_APPLICATION; + } + builder.put(REMOTE_APPLICATION, remoteApplication); + } + + private static void setRemoteOperation( + ReadableSpan span, AttributesBuilder builder, AttributeKey remoteOperationKey) { + String remoteOperation = span.getAttribute(remoteOperationKey); + if (remoteOperation == null) { + logUnknownAttribute(REMOTE_OPERATION, span); + remoteOperation = UNKNOWN_REMOTE_OPERATION; + } + builder.put(REMOTE_OPERATION, remoteOperation); + } + + private static void logUnknownAttribute(AttributeKey attributeKey, ReadableSpan span) { + String[] params = { + attributeKey.getKey(), span.getKind().name(), span.getSpanContext().getSpanId() + }; + logger.log(Level.FINEST, "No valid {0} value found for {1} span {2}", params); + } +} diff --git a/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/MetricAttributeGenerator.java b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/MetricAttributeGenerator.java new file mode 100644 index 000000000..f978c66d8 --- /dev/null +++ b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/MetricAttributeGenerator.java @@ -0,0 +1,28 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.ReadableSpan; + +/** + * Metric attribute generator defines an interface for classes that can generate specific attributes + * to be used by a {@link SpanMetricsProcessor} to produce metrics and wrap the original span. + */ +public interface MetricAttributeGenerator { + + /** + * Given a span and associated resource, produce meaningful metric attributes for metrics produced + * from the span. If no metrics should be generated from this span, return {@link + * Attributes#empty()}. + * + * @param span - Span to be used to generate metric attributes. + * @param resource - Resource associated with Span to be used to generate metric attributes. + * @return A set of zero or more attributes. Must not return null. + */ + Attributes generateMetricAttributesFromSpan(ReadableSpan span, Resource resource); +} diff --git a/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessor.java b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessor.java new file mode 100644 index 000000000..911a8465a --- /dev/null +++ b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessor.java @@ -0,0 +1,278 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.HTTP_STATUS_CODE; +import static java.util.Objects.requireNonNull; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.DelegatingSpanData; +import io.opentelemetry.sdk.trace.data.SpanData; +import javax.annotation.concurrent.Immutable; +import org.jetbrains.annotations.Nullable; + +/** + * This processor will generate metrics based on span data. It depends on a {@link + * MetricAttributeGenerator} being provided on instantiation, which will provide a means to + * determine attributes which should be used to create metrics. Also, a delegate {@link + * SpanProcessor} must be provided, which will be used to further process and export spans. A {@link + * Resource} must also be provided, which is used to generate metrics. Finally, two {@link + * LongCounter}'s and a {@link DoubleHistogram} must be provided, which will be used to actually + * create desired metrics (see below) + * + *

SpanMetricsProcessor produces metrics for errors (e.g. HTTP 4XX status codes), faults (e.g. + * HTTP 5XX status codes), and latency (in Milliseconds). Errors and faults are counted, while + * latency is measured with a histogram. Metrics are emitted with attributes derived from span + * attributes, and these derived attributes will also be added to the span passed to the delegate + * SpanProcessor. + * + *

For highest fidelity metrics, this processor should be coupled with the {@link + * AlwaysRecordSampler}, which will result in 100% of spans being sent to the processor. + */ +@Immutable +public final class SpanMetricsProcessor implements SpanProcessor { + + private static final double NANOS_TO_MILLIS = 1_000_000.0; + + // Constants for deriving error and fault metrics + private static final int ERROR_CODE_LOWER_BOUND = 400; + private static final int ERROR_CODE_UPPER_BOUND = 499; + private static final int FAULT_CODE_LOWER_BOUND = 500; + private static final int FAULT_CODE_UPPER_BOUND = 599; + + // Metric instruments + private final LongCounter errorCounter; + private final LongCounter faultCounter; + private final DoubleHistogram latencyHistogram; + + private final MetricAttributeGenerator generator; + private final SpanProcessor delegate; + private final Resource resource; + + public static SpanMetricsProcessor create( + LongCounter errorCounter, + LongCounter faultCounter, + DoubleHistogram latencyHistogram, + MetricAttributeGenerator generator, + SpanProcessor delegate, + Resource resource) { + return new SpanMetricsProcessor( + errorCounter, faultCounter, latencyHistogram, generator, delegate, resource); + } + + private SpanMetricsProcessor( + LongCounter errorCounter, + LongCounter faultCounter, + DoubleHistogram latencyHistogram, + MetricAttributeGenerator generator, + SpanProcessor delegate, + Resource resource) { + this.errorCounter = errorCounter; + this.faultCounter = faultCounter; + this.latencyHistogram = latencyHistogram; + this.generator = generator; + this.delegate = delegate; + this.resource = resource; + } + + @Override + public void onStart(Context parentContext, ReadWriteSpan span) { + delegate.onStart(parentContext, span); + } + + @Override + public boolean isStartRequired() { + return delegate.isStartRequired(); + } + + @Override + public void onEnd(ReadableSpan span) { + Attributes attributes = generator.generateMetricAttributesFromSpan(span, resource); + + // Only record metrics if non-empty attributes are returned. + if (!attributes.isEmpty()) { + recordErrorOrFault(span, attributes); + recordLatency(span, attributes); + } + + if (delegate.isEndRequired()) { + // Only wrap the span if we need to (i.e. if it will be exported and if there is anything to + // wrap it with). + if (span.getSpanContext().isSampled() && !attributes.isEmpty()) { + span = wrapSpanWithAttributes(span, attributes); + } + delegate.onEnd(span); + } + } + + @Override + public boolean isEndRequired() { + return true; + } + + @Override + public CompletableResultCode shutdown() { + return delegate.shutdown(); + } + + @Override + public CompletableResultCode forceFlush() { + return delegate.forceFlush(); + } + + @Override + public void close() { + delegate.close(); + } + + private void recordErrorOrFault(ReadableSpan span, Attributes attributes) { + Long httpStatusCode = span.getAttribute(HTTP_STATUS_CODE); + if (httpStatusCode == null) { + return; + } + + if (httpStatusCode >= ERROR_CODE_LOWER_BOUND && httpStatusCode <= ERROR_CODE_UPPER_BOUND) { + errorCounter.add(1, attributes); + } else if (httpStatusCode >= FAULT_CODE_LOWER_BOUND + && httpStatusCode <= FAULT_CODE_UPPER_BOUND) { + faultCounter.add(1, attributes); + } + } + + private void recordLatency(ReadableSpan span, Attributes attributes) { + long nanos = span.getLatencyNanos(); + double millis = nanos / NANOS_TO_MILLIS; + latencyHistogram.record(millis, attributes); + } + + /** + * {@link #onEnd} works with a {@link ReadableSpan}, which does not permit modification. However, + * we need to add derived metric attributes to the span for downstream metric to span correlation. + * To work around this, we will wrap the ReadableSpan with a {@link DelegatingReadableSpan} that + * simply passes through all API calls, except for those pertaining to Attributes, i.e {@link + * ReadableSpan#toSpanData()} and {@link ReadableSpan#getAttribute} APIs. + * + *

Note that this approach relies on {@link DelegatingSpanData} to wrap toSpanData. + * Unfortunately, no such wrapper appears to exist for ReadableSpan, so we use a new inner class, + * {@link DelegatingReadableSpan}. + * + *

See https://github.com/open-telemetry/opentelemetry-specification/issues/1089 for more + * context on this approach. + */ + private ReadableSpan wrapSpanWithAttributes(ReadableSpan span, Attributes metricAttributes) { + SpanData spanData = span.toSpanData(); + Attributes originalAttributes = spanData.getAttributes(); + Attributes replacementAttributes = + originalAttributes.toBuilder().putAll(metricAttributes).build(); + + int originalTotalAttributeCount = spanData.getTotalAttributeCount(); + int replacementTotalAttributeCount = originalTotalAttributeCount + metricAttributes.size(); + + return new DelegatingReadableSpan(span) { + @Override + public SpanData toSpanData() { + return new DelegatingSpanData(spanData) { + @Override + public Attributes getAttributes() { + return replacementAttributes; + } + + @Override + public int getTotalAttributeCount() { + return replacementTotalAttributeCount; + } + }; + } + + @Nullable + @Override + public T getAttribute(AttributeKey key) { + T attributeValue = span.getAttribute(key); + if (attributeValue != null) { + return attributeValue; + } else { + return metricAttributes.get(key); + } + } + }; + } + + /** + * A {@link ReadableSpan} which delegates all methods to another {@link ReadableSpan}. We extend + * this class to modify the {@link ReadableSpan} that will be processed by the {@link #delegate}. + */ + private class DelegatingReadableSpan implements ReadableSpan { + private final ReadableSpan delegate; + + protected DelegatingReadableSpan(ReadableSpan delegate) { + this.delegate = requireNonNull(delegate, "delegate"); + } + + @Override + public SpanContext getSpanContext() { + return delegate.getSpanContext(); + } + + @Override + public SpanContext getParentSpanContext() { + return delegate.getParentSpanContext(); + } + + @Override + public String getName() { + return delegate.getName(); + } + + @Override + public SpanData toSpanData() { + return delegate.toSpanData(); + } + + @Override + @Deprecated + public io.opentelemetry.sdk.common.InstrumentationLibraryInfo getInstrumentationLibraryInfo() { + return delegate.getInstrumentationLibraryInfo(); + } + + @Override + public InstrumentationScopeInfo getInstrumentationScopeInfo() { + return delegate.getInstrumentationScopeInfo(); + } + + @Override + public boolean hasEnded() { + return delegate.hasEnded(); + } + + @Override + public long getLatencyNanos() { + return delegate.getLatencyNanos(); + } + + @Override + public SpanKind getKind() { + return delegate.getKind(); + } + + @Nullable + @Override + public T getAttribute(AttributeKey key) { + return delegate.getAttribute(key); + } + } +} diff --git a/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorBuilder.java b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorBuilder.java new file mode 100644 index 000000000..6ad74b79d --- /dev/null +++ b/aws-xray/src/main/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorBuilder.java @@ -0,0 +1,85 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import static java.util.Objects.requireNonNull; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SpanProcessor; + +/** A builder for {@link SpanMetricsProcessor} */ +public final class SpanMetricsProcessorBuilder { + + // Metric instrument configuration constants + private static final String ERROR = "Error"; + private static final String FAULT = "Fault"; + private static final String LATENCY = "Latency"; + private static final String LATENCY_UNITS = "Milliseconds"; + + // Defaults + private static final MetricAttributeGenerator DEFAULT_GENERATOR = + new AwsMetricAttributeGenerator(); + private static final String DEFAULT_SCOPE_NAME = "SpanMetricsProcessor"; + + // Required builder elements + private final MeterProvider meterProvider; + private final SpanProcessor delegate; + private final Resource resource; + + // Optional builder elements + private MetricAttributeGenerator generator = DEFAULT_GENERATOR; + private String scopeName = DEFAULT_SCOPE_NAME; + + public static SpanMetricsProcessorBuilder create( + MeterProvider meterProvider, SpanProcessor delegate, Resource resource) { + return new SpanMetricsProcessorBuilder(meterProvider, delegate, resource); + } + + private SpanMetricsProcessorBuilder( + MeterProvider meterProvider, SpanProcessor delegate, Resource resource) { + this.meterProvider = meterProvider; + this.delegate = delegate; + this.resource = resource; + } + + /** + * Sets the generator used to generate attributes used in metrics produced by span metrics + * processor. If unset, defaults to {@link #DEFAULT_GENERATOR}. Must not be null. + */ + @CanIgnoreReturnValue + public SpanMetricsProcessorBuilder setGenerator(MetricAttributeGenerator generator) { + requireNonNull(generator, "generator"); + this.generator = generator; + return this; + } + + /** + * Sets the scope name used in the creation of metrics by the span metrics processor. If unset, + * defaults to {@link #DEFAULT_SCOPE_NAME}. Must not be null. + */ + @CanIgnoreReturnValue + public SpanMetricsProcessorBuilder setScopeName(String scopeName) { + requireNonNull(scopeName, "scopeName"); + this.scopeName = scopeName; + return this; + } + + public SpanMetricsProcessor build() { + Meter meter = meterProvider.get(scopeName); + LongCounter errorCounter = meter.counterBuilder(ERROR).build(); + LongCounter faultCounter = meter.counterBuilder(FAULT).build(); + DoubleHistogram latencyHistogram = + meter.histogramBuilder(LATENCY).setUnit(LATENCY_UNITS).build(); + + return SpanMetricsProcessor.create( + errorCounter, faultCounter, latencyHistogram, generator, delegate, resource); + } +} diff --git a/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSamplerTest.java b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSamplerTest.java new file mode 100644 index 000000000..42093c51f --- /dev/null +++ b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AlwaysRecordSamplerTest.java @@ -0,0 +1,108 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceId; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.Collections; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link AlwaysRecordSampler}. */ +class AlwaysRecordSamplerTest { + + // Mocks + private Sampler mockSampler; + + private AlwaysRecordSampler sampler; + + @BeforeEach + void setUpSamplers() { + mockSampler = mock(Sampler.class); + sampler = AlwaysRecordSampler.create(mockSampler); + } + + @Test + void testGetDescription() { + when(mockSampler.getDescription()).thenReturn("mockDescription"); + assertThat(sampler.getDescription()).isEqualTo("AlwaysRecordSampler{mockDescription}"); + } + + @Test + void testRecordAndSampleSamplingDecision() { + validateShouldSample(SamplingDecision.RECORD_AND_SAMPLE, SamplingDecision.RECORD_AND_SAMPLE); + } + + @Test + void testRecordOnlySamplingDecision() { + validateShouldSample(SamplingDecision.RECORD_ONLY, SamplingDecision.RECORD_ONLY); + } + + @Test + void testDropSamplingDecision() { + validateShouldSample(SamplingDecision.DROP, SamplingDecision.RECORD_ONLY); + } + + private void validateShouldSample( + SamplingDecision rootDecision, SamplingDecision expectedDecision) { + SamplingResult rootResult = buildRootSamplingResult(rootDecision); + when(mockSampler.shouldSample(any(), anyString(), anyString(), any(), any(), any())) + .thenReturn(rootResult); + SamplingResult actualResult = + sampler.shouldSample( + Context.current(), + TraceId.fromLongs(1, 2), + "name", + SpanKind.CLIENT, + Attributes.empty(), + Collections.emptyList()); + + if (rootDecision.equals(expectedDecision)) { + assertThat(actualResult).isEqualTo(rootResult); + assertThat(actualResult.getDecision()).isEqualTo(rootDecision); + } else { + assertThat(actualResult).isNotEqualTo(rootResult); + assertThat(actualResult.getDecision()).isEqualTo(expectedDecision); + } + + assertThat(actualResult.getAttributes()).isEqualTo(rootResult.getAttributes()); + TraceState traceState = TraceState.builder().build(); + assertThat(actualResult.getUpdatedTraceState(traceState)) + .isEqualTo(rootResult.getUpdatedTraceState(traceState)); + } + + private static SamplingResult buildRootSamplingResult(SamplingDecision samplingDecision) { + return new SamplingResult() { + @Override + public SamplingDecision getDecision() { + return samplingDecision; + } + + @Override + public Attributes getAttributes() { + return Attributes.of(AttributeKey.stringKey("key"), samplingDecision.name()); + } + + @Override + public TraceState getUpdatedTraceState(TraceState parentTraceState) { + return TraceState.builder().put("key", samplingDecision.name()).build(); + } + }; + } +} diff --git a/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGeneratorTest.java b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGeneratorTest.java new file mode 100644 index 000000000..76587735b --- /dev/null +++ b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/AwsMetricAttributeGeneratorTest.java @@ -0,0 +1,321 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import static io.opentelemetry.contrib.awsxray.AwsAttributeKeys.AWS_LOCAL_OPERATION; +import static io.opentelemetry.contrib.awsxray.AwsAttributeKeys.AWS_REMOTE_APPLICATION; +import static io.opentelemetry.contrib.awsxray.AwsAttributeKeys.AWS_REMOTE_OPERATION; +import static io.opentelemetry.semconv.resource.attributes.ResourceAttributes.SERVICE_NAME; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.DB_OPERATION; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.DB_SYSTEM; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.FAAS_INVOKED_NAME; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.FAAS_INVOKED_PROVIDER; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.GRAPHQL_OPERATION_TYPE; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.MESSAGING_OPERATION; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.MESSAGING_SYSTEM; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.PEER_SERVICE; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.RPC_METHOD; +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.RPC_SERVICE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.ReadableSpan; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** Unit tests for {@link AwsMetricAttributeGenerator}. */ +class AwsMetricAttributeGeneratorTest { + + private static final AwsMetricAttributeGenerator GENERATOR = new AwsMetricAttributeGenerator(); + + // String constants that are used many times in these tests. + private static final AttributeKey APPLICATION = AttributeKey.stringKey("Application"); + private static final AttributeKey OPERATION = AttributeKey.stringKey("Operation"); + private static final AttributeKey REMOTE_APPLICATION = + AttributeKey.stringKey("RemoteApplication"); + private static final AttributeKey REMOTE_OPERATION = + AttributeKey.stringKey("RemoteOperation"); + private static final AttributeKey SPAN_KIND = AttributeKey.stringKey("span.kind"); + private static final String AWS_LOCAL_OPERATION_VALUE = "AWS local operation"; + private static final String AWS_REMOTE_APPLICATION_VALUE = "AWS remote application"; + private static final String AWS_REMOTE_OPERATION_VALUE = "AWS remote operation"; + private static final String SERVICE_NAME_VALUE = "Service name"; + private static final String SPAN_NAME_VALUE = "Span name"; + private static final String UNKNOWN_APPLICATION = "UnknownApplication"; + private static final String UNKNOWN_OPERATION = "UnknownOperation"; + private static final String UNKNOWN_REMOTE_APPLICATION = "UnknownRemoteApplication"; + private static final String UNKNOWN_REMOTE_OPERATION = "UnknownRemoteOperation"; + + private ReadableSpan readableSpanMock; + private Resource resource; + + @BeforeEach + public void setUpMocks() { + readableSpanMock = mock(ReadableSpan.class); + when(readableSpanMock.getSpanContext()).thenReturn(mock(SpanContext.class)); + + resource = Resource.empty(); + } + + @Test + public void testConsumerSpanWithoutAttributes() { + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.CONSUMER.name(), + APPLICATION, UNKNOWN_APPLICATION, + OPERATION, UNKNOWN_OPERATION); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.CONSUMER); + } + + @Test + public void testServerSpanWithoutAttributes() { + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.SERVER.name(), + APPLICATION, UNKNOWN_APPLICATION, + OPERATION, UNKNOWN_OPERATION); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.SERVER); + } + + @Test + public void testProducerSpanWithoutAttributes() { + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.PRODUCER.name(), + APPLICATION, UNKNOWN_APPLICATION, + OPERATION, UNKNOWN_OPERATION, + REMOTE_APPLICATION, UNKNOWN_REMOTE_APPLICATION, + REMOTE_OPERATION, UNKNOWN_REMOTE_OPERATION); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.PRODUCER); + } + + @Test + public void testClientSpanWithoutAttributes() { + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.CLIENT.name(), + APPLICATION, UNKNOWN_APPLICATION, + OPERATION, UNKNOWN_OPERATION, + REMOTE_APPLICATION, UNKNOWN_REMOTE_APPLICATION, + REMOTE_OPERATION, UNKNOWN_REMOTE_OPERATION); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.CLIENT); + } + + @Test + public void testInternalSpan() { + // Spans with internal span kind should not produce any attributes. + validateAttributesProducedForSpanOfKind(Attributes.empty(), SpanKind.INTERNAL); + } + + @Test + public void testConsumerSpanWithAttributes() { + updateResourceWithServiceName(); + when(readableSpanMock.getName()).thenReturn(SPAN_NAME_VALUE); + + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.CONSUMER.name(), + APPLICATION, SERVICE_NAME_VALUE, + OPERATION, SPAN_NAME_VALUE); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.CONSUMER); + } + + @Test + public void testServerSpanWithAttributes() { + updateResourceWithServiceName(); + when(readableSpanMock.getName()).thenReturn(SPAN_NAME_VALUE); + + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.SERVER.name(), + APPLICATION, SERVICE_NAME_VALUE, + OPERATION, SPAN_NAME_VALUE); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.SERVER); + } + + @Test + public void testProducerSpanWithAttributes() { + updateResourceWithServiceName(); + mockAttribute(AWS_LOCAL_OPERATION, AWS_LOCAL_OPERATION_VALUE); + mockAttribute(AWS_REMOTE_APPLICATION, AWS_REMOTE_APPLICATION_VALUE); + mockAttribute(AWS_REMOTE_OPERATION, AWS_REMOTE_OPERATION_VALUE); + + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.PRODUCER.name(), + APPLICATION, SERVICE_NAME_VALUE, + OPERATION, AWS_LOCAL_OPERATION_VALUE, + REMOTE_APPLICATION, AWS_REMOTE_APPLICATION_VALUE, + REMOTE_OPERATION, AWS_REMOTE_OPERATION_VALUE); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.PRODUCER); + } + + @Test + public void testClientSpanWithAttributes() { + updateResourceWithServiceName(); + mockAttribute(AWS_LOCAL_OPERATION, AWS_LOCAL_OPERATION_VALUE); + mockAttribute(AWS_REMOTE_APPLICATION, AWS_REMOTE_APPLICATION_VALUE); + mockAttribute(AWS_REMOTE_OPERATION, AWS_REMOTE_OPERATION_VALUE); + + Attributes expectedAttributes = + Attributes.of( + SPAN_KIND, SpanKind.CLIENT.name(), + APPLICATION, SERVICE_NAME_VALUE, + OPERATION, AWS_LOCAL_OPERATION_VALUE, + REMOTE_APPLICATION, AWS_REMOTE_APPLICATION_VALUE, + REMOTE_OPERATION, AWS_REMOTE_OPERATION_VALUE); + validateAttributesProducedForSpanOfKind(expectedAttributes, SpanKind.CLIENT); + } + + @Test + public void testRemoteAttributesCombinations() { + // Set all expected fields to a test string, we will overwrite them in descending order to test + // the priority-order logic in AwsMetricAttributeGenerator remote attribute methods. + mockAttribute(AWS_REMOTE_APPLICATION, "TestString"); + mockAttribute(AWS_REMOTE_OPERATION, "TestString"); + mockAttribute(RPC_SERVICE, "TestString"); + mockAttribute(RPC_METHOD, "TestString"); + mockAttribute(DB_SYSTEM, "TestString"); + mockAttribute(DB_OPERATION, "TestString"); + mockAttribute(FAAS_INVOKED_PROVIDER, "TestString"); + mockAttribute(FAAS_INVOKED_NAME, "TestString"); + mockAttribute(MESSAGING_SYSTEM, "TestString"); + mockAttribute(MESSAGING_OPERATION, "TestString"); + mockAttribute(GRAPHQL_OPERATION_TYPE, "TestString"); + // Do not set dummy value for PEER_SERVICE, since it has special behaviour. + + // Two unused attributes to show that we will not make use of unrecognized attributes + mockAttribute(AttributeKey.stringKey("unknown.application.key"), "TestString"); + mockAttribute(AttributeKey.stringKey("unknown.operation.key"), "TestString"); + + // Validate behaviour of various combinations of AWS remote attributes, then remove them. + validateAndRemoveRemoteAttributes( + AWS_REMOTE_APPLICATION, + AWS_REMOTE_APPLICATION_VALUE, + AWS_REMOTE_OPERATION, + AWS_REMOTE_OPERATION_VALUE); + + // Validate behaviour of various combinations of RPC attributes, then remove them. + validateAndRemoveRemoteAttributes(RPC_SERVICE, "RPC service", RPC_METHOD, "RPC method"); + + // Validate behaviour of various combinations of DB attributes, then remove them. + validateAndRemoveRemoteAttributes(DB_SYSTEM, "DB system", DB_OPERATION, "DB operation"); + + // Validate behaviour of various combinations of FAAS attributes, then remove them. + validateAndRemoveRemoteAttributes( + FAAS_INVOKED_PROVIDER, "FAAS invoked provider", FAAS_INVOKED_NAME, "FAAS invoked name"); + + // Validate behaviour of various combinations of Messaging attributes, then remove them. + validateAndRemoveRemoteAttributes( + MESSAGING_SYSTEM, "Messaging system", MESSAGING_OPERATION, "Messaging operation"); + + // Validate behaviour of GraphQL operation type attribute, then remove it. + mockAttribute(GRAPHQL_OPERATION_TYPE, "GraphQL operation type"); + validateExpectedRemoteAttributes("graphql", "GraphQL operation type"); + mockAttribute(GRAPHQL_OPERATION_TYPE, null); + + // Validate behaviour of Peer service attribute, then remove it. + mockAttribute(PEER_SERVICE, "Peer service"); + validateExpectedRemoteAttributes("Peer service", UNKNOWN_REMOTE_OPERATION); + mockAttribute(PEER_SERVICE, null); + + // Once we have removed all usable metrics, we only have "unknown" attributes, which are unused. + validateExpectedRemoteAttributes(UNKNOWN_REMOTE_APPLICATION, UNKNOWN_REMOTE_OPERATION); + } + + @Test + public void testPeerServiceDoesOverrideOtherRemoteApplications() { + validatePeerServiceDoesOverride(RPC_SERVICE); + validatePeerServiceDoesOverride(DB_SYSTEM); + validatePeerServiceDoesOverride(FAAS_INVOKED_PROVIDER); + validatePeerServiceDoesOverride(MESSAGING_SYSTEM); + validatePeerServiceDoesOverride(GRAPHQL_OPERATION_TYPE); + // Actually testing that peer service overrides "UnknownRemoteApplication". + validatePeerServiceDoesOverride(AttributeKey.stringKey("unknown.application.key")); + } + + @Test + public void testPeerServiceDoesNotOverrideAwsRemoteApplication() { + mockAttribute(AWS_REMOTE_APPLICATION, "TestString"); + mockAttribute(PEER_SERVICE, "PeerService"); + + when(readableSpanMock.getKind()).thenReturn(SpanKind.CLIENT); + Attributes actualAttributes = + GENERATOR.generateMetricAttributesFromSpan(readableSpanMock, resource); + assertThat(actualAttributes.get(REMOTE_APPLICATION)).isEqualTo("TestString"); + } + + private void mockAttribute(AttributeKey key, String value) { + when(readableSpanMock.getAttribute(key)).thenReturn(value); + } + + private void validateAttributesProducedForSpanOfKind( + Attributes expectedAttributes, SpanKind kind) { + when(readableSpanMock.getKind()).thenReturn(kind); + Attributes actualAttributes = + GENERATOR.generateMetricAttributesFromSpan(readableSpanMock, resource); + assertThat(actualAttributes).isEqualTo(expectedAttributes); + } + + private void updateResourceWithServiceName() { + resource = Resource.builder().put(SERVICE_NAME, SERVICE_NAME_VALUE).build(); + } + + private void validateExpectedRemoteAttributes( + String expectedRemoteApplication, String expectedRemoteOperation) { + when(readableSpanMock.getKind()).thenReturn(SpanKind.CLIENT); + Attributes actualAttributes = + GENERATOR.generateMetricAttributesFromSpan(readableSpanMock, resource); + assertThat(actualAttributes.get(REMOTE_APPLICATION)).isEqualTo(expectedRemoteApplication); + assertThat(actualAttributes.get(REMOTE_OPERATION)).isEqualTo(expectedRemoteOperation); + + when(readableSpanMock.getKind()).thenReturn(SpanKind.PRODUCER); + actualAttributes = GENERATOR.generateMetricAttributesFromSpan(readableSpanMock, resource); + assertThat(actualAttributes.get(REMOTE_APPLICATION)).isEqualTo(expectedRemoteApplication); + assertThat(actualAttributes.get(REMOTE_OPERATION)).isEqualTo(expectedRemoteOperation); + } + + private void validateAndRemoveRemoteAttributes( + AttributeKey remoteApplicationKey, + String remoteApplicationValue, + AttributeKey remoteOperationKey, + String remoteOperationValue) { + mockAttribute(remoteApplicationKey, remoteApplicationValue); + mockAttribute(remoteOperationKey, remoteOperationValue); + validateExpectedRemoteAttributes(remoteApplicationValue, remoteOperationValue); + + mockAttribute(remoteApplicationKey, null); + mockAttribute(remoteOperationKey, remoteOperationValue); + validateExpectedRemoteAttributes(UNKNOWN_REMOTE_APPLICATION, remoteOperationValue); + + mockAttribute(remoteApplicationKey, remoteApplicationValue); + mockAttribute(remoteOperationKey, null); + validateExpectedRemoteAttributes(remoteApplicationValue, UNKNOWN_REMOTE_OPERATION); + + mockAttribute(remoteApplicationKey, null); + mockAttribute(remoteOperationKey, null); + } + + private void validatePeerServiceDoesOverride(AttributeKey remoteApplicationKey) { + mockAttribute(remoteApplicationKey, "TestString"); + mockAttribute(PEER_SERVICE, "PeerService"); + + // Validate that peer service value takes precedence over whatever remoteApplicationKey was set + when(readableSpanMock.getKind()).thenReturn(SpanKind.CLIENT); + Attributes actualAttributes = + GENERATOR.generateMetricAttributesFromSpan(readableSpanMock, resource); + assertThat(actualAttributes.get(REMOTE_APPLICATION)).isEqualTo("PeerService"); + + mockAttribute(remoteApplicationKey, null); + mockAttribute(PEER_SERVICE, null); + } +} diff --git a/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorTest.java b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorTest.java new file mode 100644 index 000000000..f157d1bb7 --- /dev/null +++ b/aws-xray/src/test/java/io/opentelemetry/contrib/awsxray/SpanMetricsProcessorTest.java @@ -0,0 +1,409 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.awsxray; + +import static io.opentelemetry.semconv.trace.attributes.SemanticAttributes.HTTP_STATUS_CODE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.SpanData; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +/** Unit tests for {@link SpanMetricsProcessor}. */ +class SpanMetricsProcessorTest { + + // Test constants + private static final boolean IS_SAMPLED = true; + private static final boolean IS_NOT_SAMPLED = false; + private static final boolean START_REQUIRED = true; + private static final boolean START_NOT_REQUIRED = false; + private static final boolean END_REQUIRED = true; + private static final boolean END_NOT_REQUIRED = false; + private static final boolean CONTAINS_ATTRIBUTES = true; + private static final boolean CONTAINS_NO_ATTRIBUTES = false; + private static final double TEST_LATENCY_MILLIS = 150.0; + private static final long TEST_LATENCY_NANOS = 150_000_000L; + + // Resource is not mockable, but tests can safely rely on an empty resource. + private static final Resource testResource = Resource.empty(); + + // Useful enum for indicating expected HTTP status code-related metrics + private enum ExpectedStatusMetric { + ERROR, + FAULT, + NEITHER + } + + // Mocks required for tests. + private LongCounter errorCounterMock; + private LongCounter faultCounterMock; + private DoubleHistogram latencyHistogramMock; + private MetricAttributeGenerator generatorMock; + private SpanProcessor delegateMock; + + private SpanMetricsProcessor spanMetricsProcessor; + + @BeforeEach + public void setUpMocks() { + errorCounterMock = mock(LongCounter.class); + faultCounterMock = mock(LongCounter.class); + latencyHistogramMock = mock(DoubleHistogram.class); + generatorMock = mock(MetricAttributeGenerator.class); + delegateMock = mock(SpanProcessor.class); + + spanMetricsProcessor = + SpanMetricsProcessor.create( + errorCounterMock, + faultCounterMock, + latencyHistogramMock, + generatorMock, + delegateMock, + testResource); + } + + @Test + public void testIsRequired() { + // Start requirement is dependent on the delegate + when(delegateMock.isStartRequired()).thenReturn(START_REQUIRED); + assertThat(spanMetricsProcessor.isStartRequired()).isTrue(); + when(delegateMock.isStartRequired()).thenReturn(START_NOT_REQUIRED); + assertThat(spanMetricsProcessor.isStartRequired()).isFalse(); + verify(delegateMock, times(2)).isStartRequired(); + + // End requirement is always required. + when(delegateMock.isEndRequired()).thenReturn(END_REQUIRED); + assertThat(spanMetricsProcessor.isEndRequired()).isTrue(); + when(delegateMock.isEndRequired()).thenReturn(END_NOT_REQUIRED); + assertThat(spanMetricsProcessor.isEndRequired()).isTrue(); + verify(delegateMock, never()).isEndRequired(); + } + + @Test + public void testPassthroughDelegations() { + Context parentContextMock = mock(Context.class); + ReadWriteSpan spanMock = mock(ReadWriteSpan.class); + spanMetricsProcessor.onStart(parentContextMock, spanMock); + spanMetricsProcessor.shutdown(); + spanMetricsProcessor.forceFlush(); + spanMetricsProcessor.close(); + verify(delegateMock, times(1)).onStart(eq(parentContextMock), eq(spanMock)); + verify(delegateMock, times(1)).shutdown(); + verify(delegateMock, times(1)).forceFlush(); + verify(delegateMock, times(1)).close(); + } + + /** + * Tests starting with testOnEndDelegation are testing the delegation logic of onEnd - i.e. the + * logic in SpanMetricsProcessor pertaining to calling its delegate SpanProcessors' onEnd method. + */ + @Test + public void testOnEndDelegationWithoutEndRequired() { + Attributes spanAttributes = buildSpanAttributes(CONTAINS_ATTRIBUTES); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_NOT_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + verify(delegateMock, never()).onEnd(any()); + } + + @Test + public void testOnEndDelegationWithoutMetricAttributes() { + Attributes spanAttributes = buildSpanAttributes(CONTAINS_ATTRIBUTES); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_NO_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + verify(delegateMock, times(1)).onEnd(eq(readableSpanMock)); + } + + @Test + public void testOnEndDelegationWithoutSampling() { + Attributes spanAttributes = buildSpanAttributes(CONTAINS_ATTRIBUTES); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_NOT_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + verify(delegateMock, times(1)).onEnd(eq(readableSpanMock)); + } + + @Test + public void testOnEndDelegationWithoutSpanAttributes() { + Attributes spanAttributes = buildSpanAttributes(CONTAINS_NO_ATTRIBUTES); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + ArgumentCaptor readableSpanCaptor = ArgumentCaptor.forClass(ReadableSpan.class); + verify(delegateMock, times(1)).onEnd(readableSpanCaptor.capture()); + ReadableSpan delegateSpan = readableSpanCaptor.getValue(); + assertThat(delegateSpan).isNotEqualTo(readableSpanMock); + + metricAttributes.forEach((k, v) -> assertThat(delegateSpan.getAttribute(k)).isEqualTo(v)); + + SpanData delegateSpanData = delegateSpan.toSpanData(); + Attributes delegateAttributes = delegateSpanData.getAttributes(); + assertThat(delegateAttributes.size()).isEqualTo(metricAttributes.size()); + assertThat(delegateSpanData.getTotalAttributeCount()).isEqualTo(metricAttributes.size()); + metricAttributes.forEach((k, v) -> assertThat(delegateAttributes.get(k)).isEqualTo(v)); + } + + @Test + public void testOnEndDelegationWithEverything() { + Attributes spanAttributes = buildSpanAttributes(CONTAINS_ATTRIBUTES); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + ArgumentCaptor readableSpanCaptor = ArgumentCaptor.forClass(ReadableSpan.class); + verify(delegateMock, times(1)).onEnd(readableSpanCaptor.capture()); + ReadableSpan delegateSpan = readableSpanCaptor.getValue(); + assertThat(delegateSpan).isNotEqualTo(readableSpanMock); + + spanAttributes.forEach((k, v) -> assertThat(delegateSpan.getAttribute(k)).isEqualTo(v)); + metricAttributes.forEach((k, v) -> assertThat(delegateSpan.getAttribute(k)).isEqualTo(v)); + + SpanData delegateSpanData = delegateSpan.toSpanData(); + Attributes delegateAttributes = delegateSpanData.getAttributes(); + assertThat(delegateAttributes.size()) + .isEqualTo(metricAttributes.size() + spanAttributes.size()); + assertThat(delegateSpanData.getTotalAttributeCount()) + .isEqualTo(metricAttributes.size() + spanAttributes.size()); + spanAttributes.forEach((k, v) -> assertThat(delegateAttributes.get(k)).isEqualTo(v)); + metricAttributes.forEach((k, v) -> assertThat(delegateAttributes.get(k)).isEqualTo(v)); + } + + @Test + public void testOnEndDelegatedSpanBehaviour() { + Attributes spanAttributes = buildSpanAttributes(CONTAINS_ATTRIBUTES); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + ArgumentCaptor readableSpanCaptor = ArgumentCaptor.forClass(ReadableSpan.class); + verify(delegateMock, times(1)).onEnd(readableSpanCaptor.capture()); + ReadableSpan delegateSpan = readableSpanCaptor.getValue(); + assertThat(delegateSpan).isNotEqualTo(readableSpanMock); + + // Validate all calls to wrapper span get simply delegated to the wrapped span (except ones + // pertaining to attributes. + SpanContext spanContextMock = mock(SpanContext.class); + when(readableSpanMock.getSpanContext()).thenReturn(spanContextMock); + assertThat(delegateSpan.getSpanContext()).isEqualTo(spanContextMock); + + SpanContext parentSpanContextMock = mock(SpanContext.class); + when(readableSpanMock.getParentSpanContext()).thenReturn(parentSpanContextMock); + assertThat(delegateSpan.getParentSpanContext()).isEqualTo(parentSpanContextMock); + + String name = "name"; + when(readableSpanMock.getName()).thenReturn(name); + assertThat(delegateSpan.getName()).isEqualTo(name); + + // InstrumentationLibraryInfo is deprecated, so actually invoking it causes build failures. + // Excluding from this test. + + InstrumentationScopeInfo instrumentationScopeInfo = InstrumentationScopeInfo.empty(); + when(readableSpanMock.getInstrumentationScopeInfo()).thenReturn(instrumentationScopeInfo); + assertThat(delegateSpan.getInstrumentationScopeInfo()).isEqualTo(instrumentationScopeInfo); + + boolean ended = true; + when(readableSpanMock.hasEnded()).thenReturn(ended); + assertThat(delegateSpan.hasEnded()).isEqualTo(ended); + + long latencyNanos = TEST_LATENCY_NANOS; + when(readableSpanMock.getLatencyNanos()).thenReturn(latencyNanos); + assertThat(delegateSpan.getLatencyNanos()).isEqualTo(latencyNanos); + + SpanKind spanKind = SpanKind.CLIENT; + when(readableSpanMock.getKind()).thenReturn(spanKind); + assertThat(delegateSpan.getKind()).isEqualTo(spanKind); + + Long attributeValue = 0L; + when(readableSpanMock.getAttribute(HTTP_STATUS_CODE)).thenReturn(attributeValue); + assertThat(delegateSpan.getAttribute(HTTP_STATUS_CODE)).isEqualTo(attributeValue); + } + + /** + * Tests starting with testOnEndMetricsGeneration are testing the logic in SpanMetricsProcessor's + * onEnd method pertaining to metrics generation. + */ + @Test + public void testOnEndMetricsGenerationWithoutSpanAttributes() { + Attributes spanAttributes = buildSpanAttributes(CONTAINS_NO_ATTRIBUTES); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + verifyNoInteractions(errorCounterMock); + verifyNoInteractions(faultCounterMock); + verify(latencyHistogramMock, times(1)).record(eq(TEST_LATENCY_MILLIS), eq(metricAttributes)); + } + + @Test + public void testOnEndMetricsGenerationWithoutMetricAttributes() { + Attributes spanAttributes = Attributes.of(HTTP_STATUS_CODE, 500L); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_NO_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + verifyNoInteractions(errorCounterMock); + verifyNoInteractions(faultCounterMock); + verifyNoInteractions(latencyHistogramMock); + } + + @Test + public void testOnEndMetricsGenerationWithoutEndRequired() { + Attributes spanAttributes = Attributes.of(HTTP_STATUS_CODE, 500L); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_NOT_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + verifyNoInteractions(errorCounterMock); + verify(faultCounterMock, times(1)).add(eq(1L), eq(metricAttributes)); + verify(latencyHistogramMock, times(1)).record(eq(TEST_LATENCY_MILLIS), eq(metricAttributes)); + } + + @Test + public void testOnEndMetricsGenerationWithLatency() { + Attributes spanAttributes = Attributes.of(HTTP_STATUS_CODE, 200L); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + when(readableSpanMock.getLatencyNanos()).thenReturn(5_500_000L); + + spanMetricsProcessor.onEnd(readableSpanMock); + verifyNoInteractions(errorCounterMock); + verifyNoInteractions(faultCounterMock); + verify(latencyHistogramMock, times(1)).record(eq(5.5), eq(metricAttributes)); + } + + @Test + public void testOnEndMetricsGenerationWithStatusCodes() { + // Invalid HTTP status codes + validateMetricsGeneratedForHttpStatusCode(null, ExpectedStatusMetric.NEITHER); + + // Valid HTTP status codes + validateMetricsGeneratedForHttpStatusCode(200L, ExpectedStatusMetric.NEITHER); + validateMetricsGeneratedForHttpStatusCode(399L, ExpectedStatusMetric.NEITHER); + validateMetricsGeneratedForHttpStatusCode(400L, ExpectedStatusMetric.ERROR); + validateMetricsGeneratedForHttpStatusCode(499L, ExpectedStatusMetric.ERROR); + validateMetricsGeneratedForHttpStatusCode(500L, ExpectedStatusMetric.FAULT); + validateMetricsGeneratedForHttpStatusCode(599L, ExpectedStatusMetric.FAULT); + validateMetricsGeneratedForHttpStatusCode(600L, ExpectedStatusMetric.NEITHER); + } + + private static Attributes buildSpanAttributes(boolean containsAttribute) { + if (containsAttribute) { + return Attributes.of(AttributeKey.stringKey("original key"), "original value"); + } else { + return Attributes.empty(); + } + } + + private static Attributes buildMetricAttributes(boolean containsAttribute) { + if (containsAttribute) { + return Attributes.of(AttributeKey.stringKey("new key"), "new value"); + } else { + return Attributes.empty(); + } + } + + private static ReadableSpan buildReadableSpanMock(boolean isSampled, Attributes spanAttributes) { + ReadableSpan readableSpanMock = mock(ReadableSpan.class); + + // Configure latency + when(readableSpanMock.getLatencyNanos()).thenReturn(TEST_LATENCY_NANOS); + + // Configure isSampled + SpanContext spanContextMock = mock(SpanContext.class); + when(spanContextMock.isSampled()).thenReturn(isSampled); + when(readableSpanMock.getSpanContext()).thenReturn(spanContextMock); + + // Configure attributes + when(readableSpanMock.getAttribute(any())) + .thenAnswer(invocation -> spanAttributes.get(invocation.getArgument(0))); + + // Configure spanData + SpanData mockSpanData = mock(SpanData.class); + when(mockSpanData.getAttributes()).thenReturn(spanAttributes); + when(mockSpanData.getTotalAttributeCount()).thenReturn(spanAttributes.size()); + when(readableSpanMock.toSpanData()).thenReturn(mockSpanData); + + return readableSpanMock; + } + + private void configureMocksForOnEnd( + ReadableSpan readableSpanMock, boolean isEndRequired, Attributes metricAttributes) { + // Configure isEndRequired + when(delegateMock.isEndRequired()).thenReturn(isEndRequired); + + // Configure generated attributes + when(generatorMock.generateMetricAttributesFromSpan(eq(readableSpanMock), eq(testResource))) + .thenReturn(metricAttributes); + } + + private void validateMetricsGeneratedForHttpStatusCode( + Long httpStatusCode, ExpectedStatusMetric expectedStatusMetric) { + Attributes spanAttributes = Attributes.of(HTTP_STATUS_CODE, httpStatusCode); + ReadableSpan readableSpanMock = buildReadableSpanMock(IS_SAMPLED, spanAttributes); + Attributes metricAttributes = buildMetricAttributes(CONTAINS_ATTRIBUTES); + configureMocksForOnEnd(readableSpanMock, END_REQUIRED, metricAttributes); + + spanMetricsProcessor.onEnd(readableSpanMock); + switch (expectedStatusMetric) { + case ERROR: + verify(errorCounterMock, times(1)).add(eq(1L), eq(metricAttributes)); + verifyNoInteractions(faultCounterMock); + break; + case FAULT: + verifyNoInteractions(errorCounterMock); + verify(faultCounterMock, times(1)).add(eq(1L), eq(metricAttributes)); + break; + case NEITHER: + verifyNoInteractions(errorCounterMock); + verifyNoInteractions(faultCounterMock); + break; + } + + verify(latencyHistogramMock, times(1)).record(eq(TEST_LATENCY_MILLIS), eq(metricAttributes)); + + // Clear invocations so this method can be called multiple times in one test. + clearInvocations(errorCounterMock); + clearInvocations(faultCounterMock); + clearInvocations(latencyHistogramMock); + } +}