-
Notifications
You must be signed in to change notification settings - Fork 4.9k
/
testdata.go
510 lines (438 loc) · 14.8 KB
/
testdata.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package testing
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"path/filepath"
"sort"
"strings"
"testing"
"github.com/pkg/errors"
"github.com/mitchellh/hashstructure"
"gopkg.in/yaml.v2"
"github.com/elastic/beats/v7/libbeat/asset"
"github.com/elastic/beats/v7/libbeat/mapping"
"github.com/elastic/beats/v7/metricbeat/mb"
"github.com/elastic/beats/v7/metricbeat/mb/testing/flags"
"github.com/elastic/elastic-agent-libs/mapstr"
_ "github.com/elastic/beats/v7/metricbeat/include/fields"
)
const (
expectedExtension = "-expected.json"
applicationJson = "application/json"
expectedFolder = "_meta/testdata"
)
// DataConfig is the configuration for testdata tests
//
// For example for an http service that mimics the apache status page the following
// configuration could be used:
// ```
// type: http
// url: "/server-status?auto="
// suffix: plain
// omit_documented_fields_check:
// - "apache.status.hostname"
// remove_fields_from_comparison:
// - "apache.status.hostname"
// module:
// namespace: test
// ```
// A test will be run for each file with the `plain` extension in the same directory
// where a file with this configuration is placed.
type DataConfig struct {
// Path is the directory containing the read files
Path string
// WritePath is the path where to write the generated files
WritePath string
// The type of the test to run, usually `http`.
Type string
// URL of the endpoint that must be tested depending on each module
URL string
// ContentType of the data being returned by server
ContentType string `yaml:"content_type"`
// Suffix is the extension of the source file with the input contents. Defaults to `json`, `plain` is also a common use.
Suffix string
// Module is a map of specific configs that will be appended to a module configuration prior initializing it.
// For example, the following config in yaml:
// module:
// namespace: test
// foo: bar
//
// Will produce the following module config:
// - module: http
// metricsets:
// - json
// period: 10s
// hosts: ["localhost:80"]
// path: "/"
// namespace: "test"
// foo: bar
//
// (notice last two lines)
Module map[string]interface{} `yaml:"module"`
// OmitDocumentedFieldsCheck is a list of fields that must be omitted from the function that checks if the field
// is contained in {metricset}/_meta/fields.yml
OmitDocumentedFieldsCheck []string `yaml:"omit_documented_fields_check"`
// RemoveFieldsForComparison
RemoveFieldsForComparison []string `yaml:"remove_fields_from_comparison"`
}
func defaultDataConfig() DataConfig {
return DataConfig{
Path: expectedFolder,
WritePath: expectedFolder,
Suffix: "json",
ContentType: applicationJson,
}
}
// ReadDataConfig reads the testdataconfig from a path
func ReadDataConfig(t *testing.T, f string) DataConfig {
t.Helper()
config := defaultDataConfig()
configFile, err := ioutil.ReadFile(f)
if err != nil {
t.Fatalf("failed to read '%s': %v", f, err)
}
err = yaml.Unmarshal(configFile, &config)
if err != nil {
t.Fatalf("failed to parse test configuration file '%s': %v", f, err)
}
return config
}
// TestDataConfig is a convenience helper function to read the testdata config
// from the usual path
func TestDataConfig(t *testing.T) DataConfig {
t.Helper()
f := filepath.Join(expectedFolder, "config.yml")
return ReadDataConfig(t, f)
}
// TestDataFiles run tests with config from the usual path (`_meta/testdata`)
func TestDataFiles(t *testing.T, module, metricSet string) {
t.Helper()
config := TestDataConfig(t)
TestDataFilesWithConfig(t, module, metricSet, config, "")
}
// TestDataFilesWithConfig run tests for a testdata config
func TestDataFilesWithConfig(t *testing.T, module, metricSet string, config DataConfig, testPrefix string) {
t.Helper()
// the location of the read files
location := filepath.Join(config.Path, "*."+config.Suffix)
// if this function was called from data_test.go then the testPrefix is defined and not the default empty string
if testPrefix != "" {
// the prefix for read and write files should be ../../../module/moduleName/metricsetName
prefix := filepath.Join(testPrefix, module, metricSet)
location = filepath.Join(prefix, location)
// the prefix needs to be appended to the path of the expected files and the original files
config.WritePath = filepath.Join(prefix, config.WritePath)
config.Path = filepath.Join(prefix, config.Path)
}
ff, err := filepath.Glob(location)
if err != nil {
t.Fatal(err)
}
if len(ff) == 0 {
t.Fatalf("read test path without data files: %s", config.Path)
}
var files []string
for _, f := range ff {
// Exclude all the expected files
if strings.HasSuffix(f, expectedExtension) {
continue
}
files = append(files, f)
}
for _, f := range files {
t.Run(filepath.Base(f), func(t *testing.T) {
runTest(t, f, module, metricSet, config)
})
}
}
// TestMetricsetFieldsDocumented checks metricset fields are documented from metricsets that cannot run `TestDataFiles` test which contains this check
func TestMetricsetFieldsDocumented(t *testing.T, metricSet mb.MetricSet, events []mb.Event) {
var data []mapstr.M
for _, e := range events {
beatEvent := StandardizeEvent(metricSet, e, mb.AddMetricSetInfo)
data = append(data, beatEvent.Fields)
}
if err := checkDocumented(data, nil); err != nil {
t.Errorf("%v: check if fields are documented in `metricbeat/module/%s/%s/_meta/fields.yml` "+
"file or run 'make update' on Metricbeat folder to update fields in `metricbeat/fields.yml`",
err, metricSet.Module().Name(), metricSet.Name())
}
}
func runTest(t *testing.T, file string, module, metricSetName string, config DataConfig) {
filename := filepath.Base(file)
/*
If the expected suffix is '.json' we need to exclude the data.json file, since
by the end of this function we may create a data.json file if there is a docs file with the config suffix:
if strings.HasSuffix(file, "docs."+config.Suffix) {
writeDataJSON(t, data[0], filepath.Join(config.WritePath, "data.json"))
}
Since the expected file name is obtained through filename + expectedExtension, we could end up testing files like:
Metrics file: data.json
Expected metrics file: data.json-expected.json
If the config extension is '.json'.
This is not possible, since running go -test data does not produce an expected file for data.json files. This is why
we need to exclude this file from the tests.
*/
if filename == "data.json" {
return
}
t.Logf("Testing %s file\n", file)
// starts a server serving the given file under the given url
s := server(t, file, config.URL, config.ContentType)
defer s.Close()
moduleConfig := getConfig(module, metricSetName, s.URL, config)
metricSet := NewMetricSet(t, moduleConfig)
var events []mb.Event
var errs []error
switch v := metricSet.(type) {
case mb.ReportingMetricSetV2:
metricSet := NewReportingMetricSetV2(t, moduleConfig)
events, errs = ReportingFetchV2(metricSet)
case mb.ReportingMetricSetV2Error:
metricSet := NewReportingMetricSetV2Error(t, moduleConfig)
events, errs = ReportingFetchV2Error(metricSet)
default:
t.Fatalf("unknown type: %T", v)
}
// Gather errors to build also error events
for _, e := range errs {
// TODO: for errors strip out and standardise the URL error as it would create a different diff every time
events = append(events, mb.Event{Error: e})
}
var data []mapstr.M
for _, e := range events {
beatEvent := StandardizeEvent(metricSet, e, mb.AddMetricSetInfo)
// Overwrite service.address as the port changes every time
beatEvent.Fields.Put("service.address", "127.0.0.1:55555")
data = append(data, beatEvent.Fields)
}
// Sorting the events is necessary as events are not necessarily sent in the same order
sort.SliceStable(data, func(i, j int) bool {
h1, _ := hashstructure.Hash(data[i], nil)
h2, _ := hashstructure.Hash(data[j], nil)
return h1 < h2
})
if err := checkDocumented(data, config.OmitDocumentedFieldsCheck); err != nil {
t.Errorf("%v: check if fields are documented in `metricbeat/module/%s/%s/_meta/fields.yml` "+
"file or run 'make update' on Metricbeat folder to update fields in `metricbeat/fields.yml`",
err, module, metricSetName)
}
expectedFile := filepath.Join(config.WritePath, filename+expectedExtension)
// Overwrites the golden files if run with -generate
if *flags.DataFlag {
outputIndented, err := json.MarshalIndent(&data, "", " ")
if err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(expectedFile, outputIndented, 0644); err != nil {
t.Fatal(err)
}
}
// Read expected file
expected, err := ioutil.ReadFile(expectedFile)
if err != nil {
t.Fatalf("could not read file: %s", err)
}
t.Logf("Expected %s file\n", expectedFile)
expectedMap := []mapstr.M{}
if err := json.Unmarshal(expected, &expectedMap); err != nil {
t.Fatal(err)
}
for _, fieldToRemove := range config.RemoveFieldsForComparison {
for eventIndex := range data {
if err := data[eventIndex].Delete(fieldToRemove); err != nil {
t.Fatal(err)
}
}
for eventIndex := range expectedMap {
if err := expectedMap[eventIndex].Delete(fieldToRemove); err != nil {
t.Fatal(err)
}
}
}
for _, event := range data {
// ensure the event is in expected list
found := -1
for i, expectedEvent := range expectedMap {
if event.String() == expectedEvent.String() {
found = i
break
}
}
if found > -1 {
expectedMap = append(expectedMap[:found], expectedMap[found+1:]...)
} else {
t.Errorf("Event was not expected: %+v", event)
}
}
if len(expectedMap) > 0 {
t.Error("Some events were missing:")
for _, e := range expectedMap {
t.Error(e)
}
}
// If there was some error, fail before trying to write anything.
if t.Failed() {
t.FailNow()
}
if strings.HasSuffix(file, "docs."+config.Suffix) {
writeDataJSON(t, data[0], filepath.Join(config.WritePath, "data.json"))
}
}
func writeDataJSON(t *testing.T, data mapstr.M, path string) {
// Add hardcoded timestamp
data.Put("@timestamp", "2019-03-01T08:05:34.853Z")
output, err := json.MarshalIndent(&data, "", " ")
if err = ioutil.WriteFile(path, output, 0644); err != nil {
t.Fatal(err)
}
}
// checkDocumented checks that all fields which show up in the events are documented
func checkDocumented(data []mapstr.M, omitFields []string) error {
fieldsData, err := asset.GetFields("metricbeat")
if err != nil {
return err
}
fields, err := mapping.LoadFields(fieldsData)
if err != nil {
return err
}
documentedFields := fields.GetKeys()
keys := map[string]interface{}{}
for _, k := range documentedFields {
keys[k] = struct{}{}
}
for _, d := range data {
flat := d.Flatten()
if err := documentedFieldCheck(flat, keys, omitFields); err != nil {
return err
}
}
return nil
}
func documentedFieldCheck(foundKeys mapstr.M, knownKeys map[string]interface{}, omitFields []string) error {
// Sort all found keys to guarantee consistent validation messages
sortedFoundKeys := make([]string, 0, len(foundKeys))
for k := range foundKeys {
sortedFoundKeys = append(sortedFoundKeys, k)
}
sort.Strings(sortedFoundKeys)
for k := range sortedFoundKeys {
foundKey := sortedFoundKeys[k]
if _, ok := knownKeys[foundKey]; !ok {
for _, omitField := range omitFields {
if omitDocumentedField(foundKey, omitField) {
return nil
}
}
// If a field is defined as object it can also have a * somewhere
// So this checks if such a key with the * exists by testing with it
splits := strings.Split(foundKey, ".")
found := false
for pos := 1; pos < len(splits)-1; pos++ {
key := strings.Join(splits[0:pos], ".") + ".*." + strings.Join(splits[pos+1:], ".")
if _, ok := knownKeys[key]; ok {
found = true
break
}
}
if found {
continue
}
// case `status_codes.*`:
prefix := strings.Join(splits[0:len(splits)-1], ".")
if _, ok := knownKeys[prefix+".*"]; ok {
continue
}
// should cover scenarios as status_codes.*.*` and `azure.compute_vm_scaleset.*.*`
if len(splits) > 2 {
prefix = strings.Join(splits[0:len(splits)-2], ".")
if _, ok := knownKeys[prefix+".*.*"]; ok {
continue
}
}
// case `aws.*.metrics.*.*`:
if len(splits) == 5 {
if _, ok := knownKeys[splits[0]+".*."+splits[2]+".*.*"]; ok {
continue
}
}
return errors.Errorf("field missing '%s'", foundKey)
}
}
return nil
}
// omitDocumentedField returns true if 'field' is exactly like 'omitField' or if 'field' equals the prefix of 'omitField'
// if the latter contains a dot.wildcard ".*". For example:
// field: hello, omitField: world false
// field: hello, omitField: hello true
// field: elasticsearch.stats omitField: elasticsearch.stats true
// field: elasticsearch.stats.hello.world omitField: elasticsearch.* true
// field: elasticsearch.stats.hello.world omitField: * true
func omitDocumentedField(field, omitField string) bool {
if strings.Contains(omitField, "*") {
// Omit every key prefixed with chars before "*"
prefixedField := strings.Trim(omitField, ".*")
if strings.Contains(field, prefixedField) {
return true
}
} else {
// Omit only if key matches exactly
if field == omitField {
return true
}
}
return false
}
// getConfig returns config for elasticsearch module
func getConfig(module, metricSet, url string, config DataConfig) map[string]interface{} {
moduleConfig := map[string]interface{}{
"module": module,
"metricsets": []string{metricSet},
"hosts": []string{url},
}
for k, v := range config.Module {
moduleConfig[k] = v
}
return moduleConfig
}
// server starts a server with a mock output
func server(t *testing.T, path string, url string, contentType string) *httptest.Server {
body, err := ioutil.ReadFile(path)
if err != nil {
t.Fatalf("could not read file: %s", err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
query := ""
v := r.URL.Query()
if len(v) > 0 {
query += "?" + v.Encode()
}
if r.URL.Path+query == url {
w.Header().Set("Content-Type", contentType)
w.WriteHeader(200)
w.Write(body)
} else {
w.WriteHeader(404)
}
}))
return server
}