diff --git a/packages/opentelemetry-exporter-collector/src/transform.ts b/packages/opentelemetry-exporter-collector/src/transform.ts index 0e5d0c1f96..c5aca6b9dd 100644 --- a/packages/opentelemetry-exporter-collector/src/transform.ts +++ b/packages/opentelemetry-exporter-collector/src/transform.ts @@ -31,7 +31,6 @@ import { CollectorExporterConfigBase, } from './types'; import ValueType = opentelemetryProto.common.v1.ValueType; -import { InstrumentationLibrary } from '@opentelemetry/core'; /** * Converts attributes @@ -200,7 +199,7 @@ export function toCollectorExportTraceServiceRequest< T extends CollectorExporterConfigBase >( spans: ReadableSpan[], - collectorExporterBase: CollectorTraceExporterBase + collectorTraceExporterBase: CollectorTraceExporterBase ): opentelemetryProto.collector.trace.v1.ExportTraceServiceRequest { const groupedSpans: Map< Resource, @@ -209,9 +208,9 @@ export function toCollectorExportTraceServiceRequest< const additionalAttributes = Object.assign( {}, - collectorExporterBase.attributes || {}, + collectorTraceExporterBase.attributes, { - 'service.name': collectorExporterBase.serviceName, + 'service.name': collectorTraceExporterBase.serviceName, } ); @@ -246,8 +245,13 @@ export function groupSpansByResourceAndLibrary( }, new Map>()); } +/** + * Convert to InstrumentationLibrarySpans + * @param instrumentationLibrary + * @param spans + */ function toCollectorInstrumentationLibrarySpans( - instrumentationLibrary: InstrumentationLibrary, + instrumentationLibrary: core.InstrumentationLibrary, spans: ReadableSpan[] ): opentelemetryProto.trace.v1.InstrumentationLibrarySpans { return { @@ -256,6 +260,11 @@ function toCollectorInstrumentationLibrarySpans( }; } +/** + * Returns a list of resource spans which will be exported to the collector + * @param groupedSpans + * @param baseAttributes + */ function toCollectorResourceSpans( groupedSpans: Map>, baseAttributes: Attributes diff --git a/packages/opentelemetry-exporter-collector/src/transformMetrics.ts b/packages/opentelemetry-exporter-collector/src/transformMetrics.ts new file mode 100644 index 0000000000..be14cea32a --- /dev/null +++ b/packages/opentelemetry-exporter-collector/src/transformMetrics.ts @@ -0,0 +1,315 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CollectorMetricExporterBase } from './CollectorMetricExporterBase'; +import { + MetricRecord, + MetricKind, + HistogramAggregator, + MinMaxLastSumCountAggregator, + Histogram, + Distribution, +} from '@opentelemetry/metrics'; +import { opentelemetryProto, CollectorExporterConfigBase } from './types'; +import * as api from '@opentelemetry/api'; +import * as core from '@opentelemetry/core'; +import { Resource } from '@opentelemetry/resources'; +import { toCollectorResource } from './transform'; + +/** + * Converts labels + * @param labels + */ +export function toCollectorLabels( + labels: api.Labels +): opentelemetryProto.common.v1.StringKeyValue[] { + return Object.entries(labels).map(([key, value]) => { + return { key, value }; + }); +} + +/** + * Given a MetricDescriptor, return its type in a compatible format with the collector + * @param descriptor + */ +export function toCollectorType( + metric: MetricRecord +): opentelemetryProto.metrics.v1.MetricDescriptorType { + if ( + metric.descriptor.metricKind === MetricKind.COUNTER || + metric.descriptor.metricKind === MetricKind.SUM_OBSERVER + ) { + if (metric.descriptor.valueType === api.ValueType.INT) { + return opentelemetryProto.metrics.v1.MetricDescriptorType.MONOTONIC_INT64; + } + return opentelemetryProto.metrics.v1.MetricDescriptorType.MONOTONIC_DOUBLE; + } + if (metric.aggregator instanceof HistogramAggregator) { + return opentelemetryProto.metrics.v1.MetricDescriptorType.HISTOGRAM; + } + if (metric.descriptor.valueType == api.ValueType.INT) { + return opentelemetryProto.metrics.v1.MetricDescriptorType.INT64; + } + if (metric.descriptor.valueType === api.ValueType.DOUBLE) { + return opentelemetryProto.metrics.v1.MetricDescriptorType.DOUBLE; + } + + // @TODO #1294: Add Summary once implemented + return opentelemetryProto.metrics.v1.MetricDescriptorType.INVALID_TYPE; +} + +/** + * Given a MetricDescriptor, return its temporality in a compatible format with the collector + * @param descriptor + */ +export function toCollectorTemporality( + metric: MetricRecord +): opentelemetryProto.metrics.v1.MetricDescriptorTemporality { + if ( + metric.descriptor.metricKind === MetricKind.COUNTER || + metric.descriptor.metricKind === MetricKind.SUM_OBSERVER + ) { + return opentelemetryProto.metrics.v1.MetricDescriptorTemporality.CUMULATIVE; + } + if ( + metric.descriptor.metricKind === MetricKind.UP_DOWN_COUNTER || + metric.descriptor.metricKind === MetricKind.UP_DOWN_SUM_OBSERVER + ) { + return opentelemetryProto.metrics.v1.MetricDescriptorTemporality.DELTA; + } + if ( + metric.descriptor.metricKind === MetricKind.VALUE_OBSERVER || + metric.descriptor.metricKind === MetricKind.VALUE_RECORDER + ) { + // TODO: Change once LastValueAggregator is implemented. + // If the aggregator is LastValue or Exact, then it will be instantaneous + return opentelemetryProto.metrics.v1.MetricDescriptorTemporality.DELTA; + } + return opentelemetryProto.metrics.v1.MetricDescriptorTemporality + .INVALID_TEMPORALITY; +} + +/** + * Given a MetricRecord, return the Collector compatible type of MetricDescriptor + * @param metric + */ +export function toCollectorMetricDescriptor( + metric: MetricRecord +): opentelemetryProto.metrics.v1.MetricDescriptor { + return { + name: metric.descriptor.name, + description: metric.descriptor.description, + unit: metric.descriptor.unit, + type: toCollectorType(metric), + temporality: toCollectorTemporality(metric), + }; +} + +/** + * Returns an Int64Point or DoublePoint to the collector + * @param metric + * @param startTime + */ +export function toSingularPoint( + metric: MetricRecord, + startTime: number +): { + labels: opentelemetryProto.common.v1.StringKeyValue[]; + startTimeUnixNano: number; + timeUnixNano: number; + value: number; +} { + const pointValue = + metric.aggregator instanceof MinMaxLastSumCountAggregator + ? (metric.aggregator.toPoint().value as Distribution).last + : (metric.aggregator.toPoint().value as number); + + return { + labels: toCollectorLabels(metric.labels), + value: pointValue, + startTimeUnixNano: startTime, + timeUnixNano: core.hrTimeToNanoseconds( + metric.aggregator.toPoint().timestamp + ), + }; +} + +/** + * Returns a HistogramPoint to the collector + * @param metric + * @param startTime + */ +export function toHistogramPoint( + metric: MetricRecord, + startTime: number +): opentelemetryProto.metrics.v1.HistogramDataPoint { + const histValue = metric.aggregator.toPoint().value as Histogram; + return { + labels: toCollectorLabels(metric.labels), + sum: histValue.sum, + count: histValue.count, + startTimeUnixNano: startTime, + timeUnixNano: core.hrTimeToNanoseconds( + metric.aggregator.toPoint().timestamp + ), + buckets: histValue.buckets.counts.map(count => { + return { count }; + }), + explicitBounds: histValue.buckets.boundaries, + }; +} + +/** + * Converts a metric to be compatible with the collector + * @param metric + * @param startTime start time in nanoseconds + */ +export function toCollectorMetric( + metric: MetricRecord, + startTime: number +): opentelemetryProto.metrics.v1.Metric { + if ( + toCollectorType(metric) === + opentelemetryProto.metrics.v1.MetricDescriptorType.HISTOGRAM + ) { + return { + metricDescriptor: toCollectorMetricDescriptor(metric), + histogramDataPoints: [toHistogramPoint(metric, startTime)], + }; + } + if (metric.descriptor.valueType == api.ValueType.INT) { + return { + metricDescriptor: toCollectorMetricDescriptor(metric), + int64DataPoints: [toSingularPoint(metric, startTime)], + }; + } + if (metric.descriptor.valueType === api.ValueType.DOUBLE) { + return { + metricDescriptor: toCollectorMetricDescriptor(metric), + doubleDataPoints: [toSingularPoint(metric, startTime)], + }; + } // TODO: Add support for summary points once implemented + + return { + metricDescriptor: toCollectorMetricDescriptor(metric), + int64DataPoints: [], + }; +} + +/** + * Prepares metric service request to be sent to collector + * @param metrics metrics + * @param startTime start time of the metric in nanoseconds + * @param collectorMetricExporterBase + */ +export function toCollectorExportMetricServiceRequest< + T extends CollectorExporterConfigBase +>( + metrics: MetricRecord[], + startTime: number, + collectorMetricExporterBase: CollectorMetricExporterBase +): opentelemetryProto.collector.metrics.v1.ExportMetricsServiceRequest { + const groupedMetrics: Map< + Resource, + Map + > = groupMetricsByResourceAndLibrary(metrics); + const additionalAttributes = Object.assign( + {}, + collectorMetricExporterBase.attributes, + { + 'service.name': collectorMetricExporterBase.serviceName, + } + ); + return { + resourceMetrics: toCollectorResourceMetrics( + groupedMetrics, + additionalAttributes, + startTime + ), + }; +} + +/** + * Takes an array of metrics and groups them by resource and instrumentation + * library + * @param metrics metrics + */ +export function groupMetricsByResourceAndLibrary( + metrics: MetricRecord[] +): Map> { + return metrics.reduce((metricMap, metric) => { + //group by resource + let resourceMetrics = metricMap.get(metric.resource); + if (!resourceMetrics) { + resourceMetrics = new Map(); + metricMap.set(metric.resource, resourceMetrics); + } + //group by instrumentation library + let libMetrics = resourceMetrics.get(metric.instrumentationLibrary); + if (!libMetrics) { + libMetrics = new Array(); + resourceMetrics.set(metric.instrumentationLibrary, libMetrics); + } + libMetrics.push(metric); + return metricMap; + }, new Map>()); +} + +/** + * Convert to InstrumentationLibraryMetrics + * @param instrumentationLibrary + * @param metrics + * @param startTime + */ +function toCollectorInstrumentationLibraryMetrics( + instrumentationLibrary: core.InstrumentationLibrary, + metrics: MetricRecord[], + startTime: number +): opentelemetryProto.metrics.v1.InstrumentationLibraryMetrics { + return { + metrics: metrics.map(metric => toCollectorMetric(metric, startTime)), + instrumentationLibrary, + }; +} + +/** + * Returns a list of resource metrics which will be exported to the collector + * @param groupedSpans + * @param baseAttributes + */ +function toCollectorResourceMetrics( + groupedMetrics: Map< + Resource, + Map + >, + baseAttributes: api.Attributes, + startTime: number +): opentelemetryProto.metrics.v1.ResourceMetrics[] { + return Array.from(groupedMetrics, ([resource, libMetrics]) => { + return { + resource: toCollectorResource(resource, baseAttributes), + instrumentationLibraryMetrics: Array.from( + libMetrics, + ([instrumentationLibrary, metrics]) => + toCollectorInstrumentationLibraryMetrics( + instrumentationLibrary, + metrics, + startTime + ) + ), + }; + }); +} diff --git a/packages/opentelemetry-exporter-collector/src/types.ts b/packages/opentelemetry-exporter-collector/src/types.ts index 907ba60276..40eb5755d8 100644 --- a/packages/opentelemetry-exporter-collector/src/types.ts +++ b/packages/opentelemetry-exporter-collector/src/types.ts @@ -32,6 +32,11 @@ export namespace opentelemetryProto { resourceSpans: opentelemetryProto.trace.v1.ResourceSpans[]; } } + export namespace metrics.v1 { + export interface ExportMetricsServiceRequest { + resourceMetrics: opentelemetryProto.metrics.v1.ResourceMetrics[]; + } + } } export namespace resource.v1 { diff --git a/packages/opentelemetry-exporter-collector/test/common/transform.test.ts b/packages/opentelemetry-exporter-collector/test/common/transform.test.ts index 01357b2ada..2251d80a3f 100644 --- a/packages/opentelemetry-exporter-collector/test/common/transform.test.ts +++ b/packages/opentelemetry-exporter-collector/test/common/transform.test.ts @@ -26,7 +26,6 @@ import { multiInstrumentationLibraryTrace, } from '../helper'; import { Resource } from '@opentelemetry/resources'; - describe('transform', () => { describe('toCollectorAttributes', () => { it('should convert attribute string', () => { @@ -126,7 +125,6 @@ describe('transform', () => { }); }); }); - describe('groupSpansByResourceAndLibrary', () => { it('should group by resource', () => { const [resource1, resource2] = mockedResources; diff --git a/packages/opentelemetry-exporter-collector/test/common/transformMetrics.test.ts b/packages/opentelemetry-exporter-collector/test/common/transformMetrics.test.ts new file mode 100644 index 0000000000..6e4563e6b6 --- /dev/null +++ b/packages/opentelemetry-exporter-collector/test/common/transformMetrics.test.ts @@ -0,0 +1,103 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as assert from 'assert'; +import * as transform from '../../src/transformMetrics'; +import { + mockCounter, + mockObserver, + mockedResources, + mockedInstrumentationLibraries, + multiResourceMetrics, + multiInstrumentationLibraryMetrics, + ensureCounterIsCorrect, + ensureObserverIsCorrect, + mockHistogram, + ensureHistogramIsCorrect, + ensureValueRecorderIsCorrect, + mockValueRecorder, +} from '../helper'; +import { HistogramAggregator } from '@opentelemetry/metrics'; +import { hrTimeToNanoseconds } from '@opentelemetry/core'; +describe('transformMetrics', () => { + describe('toCollectorMetric', () => { + it('should convert metric', () => { + mockCounter.aggregator.update(1); + ensureCounterIsCorrect( + transform.toCollectorMetric(mockCounter, 1592602232694000000), + hrTimeToNanoseconds(mockCounter.aggregator.toPoint().timestamp) + ); + mockObserver.aggregator.update(10); + ensureObserverIsCorrect( + transform.toCollectorMetric(mockObserver, 1592602232694000000), + hrTimeToNanoseconds(mockObserver.aggregator.toPoint().timestamp) + ); + mockHistogram.aggregator.update(7); + mockHistogram.aggregator.update(14); + (mockHistogram.aggregator as HistogramAggregator).reset(); + ensureHistogramIsCorrect( + transform.toCollectorMetric(mockHistogram, 1592602232694000000), + hrTimeToNanoseconds(mockHistogram.aggregator.toPoint().timestamp) + ); + + mockValueRecorder.aggregator.update(5); + ensureValueRecorderIsCorrect( + transform.toCollectorMetric(mockValueRecorder, 1592602232694000000), + hrTimeToNanoseconds(mockValueRecorder.aggregator.toPoint().timestamp) + ); + }); + }); + describe('toCollectorMetricDescriptor', () => { + describe('groupMetricsByResourceAndLibrary', () => { + it('should group by resource', () => { + const [resource1, resource2] = mockedResources; + const [library] = mockedInstrumentationLibraries; + const [metric1, metric2, metric3] = multiResourceMetrics; + + const expected = new Map([ + [resource1, new Map([[library, [metric1, metric3]]])], + [resource2, new Map([[library, [metric2]]])], + ]); + + const result = transform.groupMetricsByResourceAndLibrary( + multiResourceMetrics + ); + + assert.deepStrictEqual(result, expected); + }); + + it('should group by instrumentation library', () => { + const [resource] = mockedResources; + const [lib1, lib2] = mockedInstrumentationLibraries; + const [metric1, metric2, metric3] = multiInstrumentationLibraryMetrics; + const expected = new Map([ + [ + resource, + new Map([ + [lib1, [metric1, metric3]], + [lib2, [metric2]], + ]), + ], + ]); + + const result = transform.groupMetricsByResourceAndLibrary( + multiInstrumentationLibraryMetrics + ); + + assert.deepStrictEqual(result, expected); + }); + }); + }); +}); diff --git a/packages/opentelemetry-exporter-collector/test/helper.ts b/packages/opentelemetry-exporter-collector/test/helper.ts index 7607beab93..f2ed9cf048 100644 --- a/packages/opentelemetry-exporter-collector/test/helper.ts +++ b/packages/opentelemetry-exporter-collector/test/helper.ts @@ -20,14 +20,15 @@ import { Resource } from '@opentelemetry/resources'; import * as assert from 'assert'; import { opentelemetryProto } from '../src/types'; import * as collectorTypes from '../src/types'; -import { InstrumentationLibrary } from '@opentelemetry/core'; -import * as grpc from 'grpc'; import { MetricRecord, MetricKind, SumAggregator, MinMaxLastSumCountAggregator, + HistogramAggregator, } from '@opentelemetry/metrics'; +import { InstrumentationLibrary } from '@opentelemetry/core'; +import * as grpc from 'grpc'; if (typeof Buffer === 'undefined') { (window as any).Buffer = { @@ -94,6 +95,42 @@ export const mockObserver: MetricRecord = { instrumentationLibrary: { name: 'default', version: '0.0.1' }, }; +export const mockValueRecorder: MetricRecord = { + descriptor: { + name: 'test-recorder', + description: 'sample recorder description', + unit: '3', + metricKind: MetricKind.VALUE_RECORDER, + valueType: ValueType.INT, + }, + labels: {}, + aggregator: new MinMaxLastSumCountAggregator(), + resource: new Resource({ + service: 'ui', + version: 1, + cost: 112.12, + }), + instrumentationLibrary: { name: 'default', version: '0.0.1' }, +}; + +export const mockHistogram: MetricRecord = { + descriptor: { + name: 'test-hist', + description: 'sample observer description', + unit: '2', + metricKind: MetricKind.VALUE_OBSERVER, + valueType: ValueType.DOUBLE, + }, + labels: {}, + aggregator: new HistogramAggregator([10, 20]), + resource: new Resource({ + service: 'ui', + version: 1, + cost: 112.12, + }), + instrumentationLibrary: { name: 'default', version: '0.0.1' }, +}; + const traceIdBase64 = 'HxAI3I4nDoXECg18OTmyeA=='; const spanIdBase64 = 'XhByYfZPpT4='; const parentIdBase64 = 'eKiRUJiGQ4g='; @@ -246,6 +283,42 @@ export const multiResourceTrace: ReadableSpan[] = [ }, ]; +export const multiResourceMetrics: MetricRecord[] = [ + { + ...mockCounter, + resource: mockedResources[0], + instrumentationLibrary: mockedInstrumentationLibraries[0], + }, + { + ...mockObserver, + resource: mockedResources[1], + instrumentationLibrary: mockedInstrumentationLibraries[0], + }, + { + ...mockCounter, + resource: mockedResources[0], + instrumentationLibrary: mockedInstrumentationLibraries[0], + }, +]; + +export const multiInstrumentationLibraryMetrics: MetricRecord[] = [ + { + ...mockCounter, + resource: mockedResources[0], + instrumentationLibrary: mockedInstrumentationLibraries[0], + }, + { + ...mockObserver, + resource: mockedResources[0], + instrumentationLibrary: mockedInstrumentationLibraries[1], + }, + { + ...mockCounter, + resource: mockedResources[0], + instrumentationLibrary: mockedInstrumentationLibraries[0], + }, +]; + export const multiInstrumentationLibraryTrace: ReadableSpan[] = [ { ...basicTrace[0], @@ -589,6 +662,101 @@ export function ensureWebResourceIsCorrect( }); } +export function ensureCounterIsCorrect( + metric: collectorTypes.opentelemetryProto.metrics.v1.Metric, + time: number +) { + assert.deepStrictEqual(metric, { + metricDescriptor: { + name: 'test-counter', + description: 'sample counter description', + unit: '1', + type: 2, + temporality: 3, + }, + int64DataPoints: [ + { + labels: [], + value: 1, + startTimeUnixNano: 1592602232694000000, + timeUnixNano: time, + }, + ], + }); +} + +export function ensureObserverIsCorrect( + metric: collectorTypes.opentelemetryProto.metrics.v1.Metric, + time: number +) { + assert.deepStrictEqual(metric, { + metricDescriptor: { + name: 'test-observer', + description: 'sample observer description', + unit: '2', + type: 3, + temporality: 2, + }, + doubleDataPoints: [ + { + labels: [], + value: 10, + startTimeUnixNano: 1592602232694000000, + timeUnixNano: time, + }, + ], + }); +} + +export function ensureValueRecorderIsCorrect( + metric: collectorTypes.opentelemetryProto.metrics.v1.Metric, + time: number +) { + assert.deepStrictEqual(metric, { + metricDescriptor: { + name: 'test-recorder', + description: 'sample recorder description', + unit: '3', + type: 1, + temporality: 2, + }, + int64DataPoints: [ + { + labels: [], + value: 5, + startTimeUnixNano: 1592602232694000000, + timeUnixNano: time, + }, + ], + }); +} + +export function ensureHistogramIsCorrect( + metric: collectorTypes.opentelemetryProto.metrics.v1.Metric, + time: number +) { + assert.deepStrictEqual(metric, { + metricDescriptor: { + name: 'test-hist', + description: 'sample observer description', + unit: '2', + type: 5, + temporality: 2, + }, + histogramDataPoints: [ + { + labels: [], + buckets: [{ count: 1 }, { count: 1 }, { count: 0 }], + count: 2, + sum: 21, + explicitBounds: [10, 20], + startTimeUnixNano: 1592602232694000000, + timeUnixNano: time, + }, + ], + }); +} + export function ensureResourceIsCorrect( resource: collectorTypes.opentelemetryProto.resource.v1.Resource ) { @@ -664,6 +832,37 @@ export function ensureExportTraceServiceRequestIsSet( assert.strictEqual(spans && spans.length, 1, 'spans are missing'); } +export function ensureExportMetricsServiceRequestIsSet( + json: collectorTypes.opentelemetryProto.collector.metrics.v1.ExportMetricsServiceRequest +) { + const resourceMetrics = json.resourceMetrics; + assert.strictEqual(resourceMetrics.length, 2, 'resourceMetrics is missing'); + + const resource = resourceMetrics[0].resource; + assert.strictEqual(!!resource, true, 'resource is missing'); + + const instrumentationLibraryMetrics = + resourceMetrics[0].instrumentationLibraryMetrics; + assert.strictEqual( + instrumentationLibraryMetrics && instrumentationLibraryMetrics.length, + 1, + 'instrumentationLibraryMetrics is missing' + ); + + const instrumentationLibrary = + instrumentationLibraryMetrics[0].instrumentationLibrary; + assert.strictEqual( + !!instrumentationLibrary, + true, + 'instrumentationLibrary is missing' + ); + + const metric1 = resourceMetrics[0].instrumentationLibraryMetrics[0].metrics; + const metric2 = resourceMetrics[1].instrumentationLibraryMetrics[0].metrics; + assert.strictEqual(metric1.length, 1, 'Metrics are missing'); + assert.strictEqual(metric2.length, 1, 'Metrics are missing'); +} + export function ensureMetadataIsCorrect( actual: grpc.Metadata, expected: grpc.Metadata