-
Notifications
You must be signed in to change notification settings - Fork 4.9k
/
pipeline.yml
96 lines (95 loc) · 3.33 KB
/
pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
description: Pipeline for parsing elasticsearch server logs
processors:
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
- rename:
field: '@timestamp'
target_field: event.created
- grok:
field: message
patterns:
- ^%{CHAR:first_char}
pattern_definitions:
CHAR: .
- pipeline:
if: ctx.first_char != '{'
name: '{< IngestPipeline "pipeline-plaintext" >}'
- pipeline:
if: ctx.first_char == '{'
name: '{< IngestPipeline "pipeline-json" >}'
- script:
lang: painless
source: >-
if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.observation_duration != null) {
if (ctx.elasticsearch.server.gc.observation_duration.unit == params.seconds_unit) {
ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_s;
}
if (ctx.elasticsearch.server.gc.observation_duration.unit == params.milliseconds_unit) {
ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time;
}
if (ctx.elasticsearch.server.gc.observation_duration.unit == params.minutes_unit) {
ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time * params.ms_in_one_m;
}
}
if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.collection_duration != null) {
if (ctx.elasticsearch.server.gc.collection_duration.unit == params.seconds_unit) {
ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_s;
}
if (ctx.elasticsearch.server.gc.collection_duration.unit == params.milliseconds_unit) {
ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time;
}
if (ctx.elasticsearch.server.gc.collection_duration.unit == params.minutes_unit) {
ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time * params.ms_in_one_m;
}
}
params:
minutes_unit: m
seconds_unit: s
milliseconds_unit: ms
ms_in_one_s: 1000
ms_in_one_m: 60000
- set:
field: event.kind
value: event
- set:
field: event.category
value: database
- script:
lang: painless
source: >-
def errorLevels = ['FATAL', 'ERROR'];
if (ctx?.log?.level != null) {
if (errorLevels.contains(ctx.log.level)) {
ctx.event.type = 'error';
} else {
ctx.event.type = 'info';
}
}
- set:
field: host.name
value: "{{elasticsearch.node.name}}"
ignore_empty_value: true
- set:
field: host.id
value: "{{elasticsearch.node.id}}"
ignore_empty_value: true
- remove:
field:
- elasticsearch.server.gc.collection_duration.time
- elasticsearch.server.gc.collection_duration.unit
- elasticsearch.server.gc.observation_duration.time
- elasticsearch.server.gc.observation_duration.unit
ignore_missing: true
- remove:
field:
- elasticsearch.server.timestamp
- elasticsearch.server.@timestamp
ignore_missing: true
- remove:
field:
- first_char
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'