From 9a3af9b7b4e3da422fcf9d8fcbac2926cbebf414 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 19 Jun 2024 15:00:17 -0700 Subject: [PATCH 1/2] update add serviceName keyword default field to exemplar Signed-off-by: YANGDB --- src/dataprepper/pipelines.yaml | 4 ++++ src/dataprepper/templates/ss4o_metrics.json | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/dataprepper/pipelines.yaml b/src/dataprepper/pipelines.yaml index 7d06cfc159..84fabf638e 100644 --- a/src/dataprepper/pipelines.yaml +++ b/src/dataprepper/pipelines.yaml @@ -293,6 +293,10 @@ otel-metrics-pipeline: "ignore_above": 256, "type": "keyword" }, + "serviceName": { + "ignore_above": 256, + "type": "keyword" + }, "spanId": { "ignore_above": 256, "type": "keyword" diff --git a/src/dataprepper/templates/ss4o_metrics.json b/src/dataprepper/templates/ss4o_metrics.json index e57c3ef150..7054bcbb30 100644 --- a/src/dataprepper/templates/ss4o_metrics.json +++ b/src/dataprepper/templates/ss4o_metrics.json @@ -211,6 +211,10 @@ "ignore_above": 256, "type": "keyword" }, + "serviceName": { + "ignore_above": 256, + "type": "keyword" + }, "spanId": { "ignore_above": 256, "type": "keyword" From 0cbae52593835fc4be761a26c6ff840d49b525bc Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 19 Jun 2024 20:57:26 -0700 Subject: [PATCH 2/2] update add serviceName keyword default field to exemplar Signed-off-by: YANGDB --- src/dataprepper/pipelines.yaml | 517 ++++++++++---------- src/dataprepper/templates/ss4o_metrics.json | 6 +- 2 files changed, 276 insertions(+), 247 deletions(-) diff --git a/src/dataprepper/pipelines.yaml b/src/dataprepper/pipelines.yaml index 84fabf638e..3a2ba9db25 100644 --- a/src/dataprepper/pipelines.yaml +++ b/src/dataprepper/pipelines.yaml @@ -84,271 +84,298 @@ otel-metrics-pipeline: bulk_size: 4 template_type: index-template template_content: > - { "template": { - "mappings": { - "_meta": { - "version": "1.0.0", - "catalog": "observability", - "type": "metrics", - "component": "metrics", - "correlations" : [ + { + "index_patterns": [ + "ss4o_metrics-*-*" + ], + "template": { + "mappings": { + "_meta": { + "version": "1.0.0", + "catalog": "observability", + "type": "metrics", + "component": "metrics", + "correlations" : [ + { + "field": "spanId", + "foreign-schema" : "traces", + "foreign-field" : "spanId" + }, + { + "field": "traceId", + "foreign-schema" : "traces", + "foreign-field" : "traceId" + } + ] + }, + "_source": { + "enabled": true + }, + "dynamic_templates": [ { - "field": "spanId", - "foreign-schema" : "traces", - "foreign-field" : "spanId" + "exemplar_attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "exemplar.attributes.*" + } }, { - "field": "traceId", - "foreign-schema" : "traces", - "foreign-field" : "traceId" - } - ] - }, - "_source": { - "enabled": true - }, - "dynamic_templates": [ - { - "exemplar_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "exemplar.attributes.*" - } - }, - { - "instrumentation_scope_attributes_map": { - "mapping": { - "type": "keyword" - }, - "path_match": "instrumentationScope.attributes.*" + "instrumentation_scope_attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "instrumentationScope.attributes.*" + } } - } - ], - "properties": { - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "attributes": { - "type": "object", - "properties": { - "data_stream": { - "properties": { - "dataset": { - "ignore_above": 128, - "type": "keyword" - }, - "namespace": { - "ignore_above": 128, - "type": "keyword" - }, - "type": { - "ignore_above": 56, - "type": "keyword" + ], + "properties": { + "serviceName": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "attributes": { + "type": "object", + "properties": { + "data_stream": { + "properties": { + "dataset": { + "ignore_above": 128, + "type": "keyword" + }, + "namespace": { + "ignore_above": 128, + "type": "keyword" + }, + "type": { + "ignore_above": 56, + "type": "keyword" + } } } } - } - }, - "description": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + }, + "description": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } } - } - }, - "unit": { - "type": "keyword", - "ignore_above": 128 - }, - "kind": { - "type": "keyword", - "ignore_above": 128 - }, - "aggregationTemporality": { - "type": "keyword", - "ignore_above": 128 - }, - "monotonic": { - "type": "boolean" - }, - "startTime": { - "type": "date" - }, - "@timestamp": { - "type": "date" - }, - "observedTimestamp": { - "type": "date_nanos" - }, - "value@int": { - "type": "integer" - }, - "value@double": { - "type": "double" - }, - "buckets": { - "type" : "nested", - "properties": { - "count": { - "type": "long" - }, - "sum": { - "type": "double" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" + }, + "unit": { + "type": "keyword", + "ignore_above": 128 + }, + "kind": { + "type": "keyword", + "ignore_above": 128 + }, + "aggregationTemporality": { + "type": "keyword", + "ignore_above": 128 + }, + "monotonic": { + "type": "boolean" + }, + "startTime": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "observedTimestamp": { + "type": "date_nanos" + }, + "value@int": { + "type": "integer" + }, + "value@double": { + "type": "double" + }, + "buckets": { + "type" : "nested", + "properties": { + "count": { + "type": "long" + }, + "sum": { + "type": "double" + }, + "max": { + "type": "float" + }, + "min": { + "type": "float" + } } - } - }, - "bucketCount": { - "type": "long" - }, - "bucketCountsList": { - "type": "long" - }, - "explicitBoundsList": { - "type": "float" - }, - "explicitBoundsCount": { - "type": "float" - }, - "quantiles": { - "properties": { - "quantile": { - "type": "double" - }, - "value": { - "type": "double" + }, + "bucketCount": { + "type": "long" + }, + "bucketCountsList": { + "type": "long" + }, + "explicitBoundsList": { + "type": "float" + }, + "explicitBoundsCount": { + "type": "float" + }, + "quantiles": { + "properties": { + "quantile": { + "type": "double" + }, + "value": { + "type": "double" + } } - } - }, - "quantileValuesCount": { - "type": "long" - }, - "positiveBuckets": { - "type" : "nested", - "properties": { - "count": { - "type": "long" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" + }, + "quantileValuesCount": { + "type": "long" + }, + "positiveBuckets": { + "type" : "nested", + "properties": { + "count": { + "type": "long" + }, + "max": { + "type": "float" + }, + "min": { + "type": "float" + } } - } - }, - "negativeBuckets": { - "type" : "nested", - "properties": { - "count": { - "type": "long" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" + }, + "negativeBuckets": { + "type" : "nested", + "properties": { + "count": { + "type": "long" + }, + "max": { + "type": "float" + }, + "min": { + "type": "float" + } } - } - }, - "negativeOffset": { - "type": "integer" - }, - "positiveOffset": { - "type": "integer" - }, - "zeroCount": { - "type": "long" - }, - "scale": { - "type": "long" - }, - "max": { - "type": "float" - }, - "min": { - "type": "float" - }, - "sum": { - "type": "float" - }, - "count": { - "type": "long" - }, - "exemplar": { - "properties": { - "time": { - "type": "date" - }, - "traceId": { - "ignore_above": 256, - "type": "keyword" - }, - "serviceName": { - "ignore_above": 256, - "type": "keyword" - }, - "spanId": { - "ignore_above": 256, - "type": "keyword" + }, + "negativeOffset": { + "type": "integer" + }, + "positiveOffset": { + "type": "integer" + }, + "zeroCount": { + "type": "long" + }, + "scale": { + "type": "long" + }, + "max": { + "type": "float" + }, + "min": { + "type": "float" + }, + "sum": { + "type": "float" + }, + "count": { + "type": "long" + }, + "exemplar": { + "properties": { + "time": { + "type": "date" + }, + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "serviceName": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + } } - } - }, - "instrumentationScope": { - "properties": { - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - }, - "droppedAttributesCount": { - "type": "integer" - }, - "schemaUrl": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + }, + "instrumentationScope": { + "properties": { + "name": { + "type": "keyword", + "ignore_above": 256 + }, + "version": { + "type": "keyword", + "ignore_above": 256 + }, + "droppedAttributesCount": { + "type": "integer" + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } } } } - } - }, - "schemaUrl": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + }, + "schemaUrl": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } } } } + }, + "aliases" : { + "otel-metrics-" : {} + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": 10000 + } + }, + "refresh_interval": "5s" + } } }, - "aliases" : { - "otel-metrics-" : {} - }, - "settings": { - "index": { - "mapping": { - "total_fields": { - "limit": 10000 - } + "composed_of": [], + "version": 1, + "_meta": { + "description": "Observability Metrics Mapping Template", + "catalog": "observability", + "type": "metrics", + "correlations" : [ + { + "field": "spanId", + "foreign-schema" : "traces", + "foreign-field" : "spanId" }, - "refresh_interval": "5s" - } + { + "field": "traceId", + "foreign-schema" : "traces", + "foreign-field" : "traceId" + } + ] } - }} \ No newline at end of file + } \ No newline at end of file diff --git a/src/dataprepper/templates/ss4o_metrics.json b/src/dataprepper/templates/ss4o_metrics.json index 7054bcbb30..c36ac47f28 100644 --- a/src/dataprepper/templates/ss4o_metrics.json +++ b/src/dataprepper/templates/ss4o_metrics.json @@ -44,6 +44,9 @@ } ], "properties": { + "serviceName": { + "type": "keyword" + }, "name": { "type": "keyword", "ignore_above": 256 @@ -289,5 +292,4 @@ } ] } -} - +} \ No newline at end of file