Skip to content

Commit

Permalink
Add index template dataprepper (#134)
Browse files Browse the repository at this point in the history
* add getting started tutorial

Signed-off-by: YANGDB <yang.db.dev@gmail.com>

* update getting started tutorial
update data-prepper with the most up to date ss4o template files

Signed-off-by: YANGDB <yang.db.dev@gmail.com>

* Update GettingStarted.md

fix tutorial redundant step

Signed-off-by: YANGDB <yang.db.dev@gmail.com>

---------

Signed-off-by: YANGDB <yang.db.dev@gmail.com>
  • Loading branch information
YANG-DB committed May 7, 2024
1 parent 1cf3548 commit df2a153
Show file tree
Hide file tree
Showing 6 changed files with 708 additions and 9 deletions.
3 changes: 3 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -846,6 +846,9 @@ services:
container_name: dataprepper
volumes:
- /data/service-map/
- ./src/dataprepper/templates/ss4o_metrics.json:/usr/share/data-prepper/templates/ss4o_metrics.json
- ./src/dataprepper/templates/ss4o_logs.json:/usr/share/data-prepper/templates/ss4o_logs.json
- ./src/dataprepper/templates/ss4o_traces.json:/usr/share/data-prepper/templates/ss4o_traces.json
- ./src/dataprepper/pipelines.yaml:/usr/share/data-prepper/pipelines/pipelines.yaml
- ./src/dataprepper/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml
ports:
Expand Down
3 changes: 2 additions & 1 deletion src/dataprepper/pipelines.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,5 +80,6 @@ otel-metrics-pipeline:
password: "my_%New%_passW0rd!@#"
insecure: true
index_type: custom
index: otel-metrics-%{yyyy.MM.dd}
template_file: "templates/ss4o_metrics.json"
index: ss4o_metrics-otel-%{yyyy.MM.dd}
bulk_size: 4
208 changes: 208 additions & 0 deletions src/dataprepper/templates/ss4o_logs.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
{
"index_patterns": ["ss4o_logs-*-*"],
"priority": 900,
"template": {
"mappings": {
"_meta": {
"version": "1.0.0",
"catalog": "observability",
"type": "logs",
"component": "log",
"correlations": [
{
"field": "spanId",
"foreign-schema": "traces",
"foreign-field": "spanId"
},
{
"field": "traceId",
"foreign-schema": "traces",
"foreign-field": "traceId"
}
]
},
"_source": {
"enabled": true
},
"dynamic_templates": [
{
"instrumentation_scope_attributes_map": {
"mapping": {
"type": "keyword"
},
"path_match": "instrumentationScope.attributes.*"
}
}
],
"properties": {
"severity": {
"properties": {
"number": {
"type": "long"
},
"text": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"attributes": {
"type": "object",
"properties": {
"data_stream": {
"properties": {
"dataset": {
"ignore_above": 128,
"type": "keyword"
},
"namespace": {
"ignore_above": 128,
"type": "keyword"
},
"type": {
"ignore_above": 56,
"type": "keyword"
}
}
}
}
},
"body": {
"type": "text"
},
"@message": {
"type": "alias",
"path": "body"
},
"@timestamp": {
"type": "date"
},
"observedTimestamp": {
"type": "date"
},
"observerTime": {
"type": "alias",
"path": "observedTimestamp"
},
"traceId": {
"ignore_above": 256,
"type": "keyword"
},
"spanId": {
"ignore_above": 256,
"type": "keyword"
},
"schemaUrl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"instrumentationScope": {
"properties": {
"name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 128
}
}
},
"version": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dropped_attributes_count": {
"type": "integer"
},
"schemaUrl": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"event": {
"properties": {
"domain": {
"ignore_above": 256,
"type": "keyword"
},
"name": {
"ignore_above": 256,
"type": "keyword"
},
"source": {
"ignore_above": 256,
"type": "keyword"
},
"category": {
"ignore_above": 256,
"type": "keyword"
},
"type": {
"ignore_above": 256,
"type": "keyword"
},
"kind": {
"ignore_above": 256,
"type": "keyword"
},
"result": {
"ignore_above": 256,
"type": "keyword"
},
"exception": {
"properties": {
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 256,
"type": "keyword"
},
"stacktrace": {
"type": "text"
}
}
}
}
}
}
},
"settings": {
"index": {
"mapping": {
"total_fields": {
"limit": 10000
}
},
"refresh_interval": "5s"
}
},
"aliases" : {
"otel-events" : {}
}
},
"composed_of": [],
"version": 1
}
Loading

0 comments on commit df2a153

Please sign in to comment.