From 88bc92f3312d2561d393e1fd5778d5b2a0ca6a4e Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Thu, 27 Sep 2018 10:39:50 +0200 Subject: [PATCH] Added 'query' parameter to metricbeats global configuration (#8292) Added 'query' parameter to metricbeats global configuration (cherry picked from commit 7dbb4f1e380c6be14eb775057a94ea4e02d49a4d) --- CHANGELOG.asciidoc | 2 ++ metricbeat/docs/metricbeat-options.asciidoc | 17 +++++++++++ metricbeat/mb/mb.go | 34 +++++++++++++++++++-- metricbeat/mb/mb_test.go | 25 ++++++++++++++- metricbeat/mb/parse/url.go | 10 ++++++ metricbeat/mb/parse/url_test.go | 3 ++ 6 files changed, 88 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 22aa8b46d16..cfa7a945dcb 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -113,6 +113,8 @@ https://github.com/elastic/beats/compare/v6.4.0...6.x[Check the HEAD diff] - Added 'died' PID state to process_system metricset on system module {pull}8275[8275] - Add `metrics` metricset to MongoDB module. {pull}7611[7611] - Added `ccr` metricset to Elasticsearch module. {pull}8335[8335] +- Added support for query params in configuration {issue}8286[8286] {pull}8292[8292] +- Support for Kafka 2.0.0 {pull}8399[8399] *Packetbeat* diff --git a/metricbeat/docs/metricbeat-options.asciidoc b/metricbeat/docs/metricbeat-options.asciidoc index 9eb64ebc0f1..802431d9ff3 100644 --- a/metricbeat/docs/metricbeat-options.asciidoc +++ b/metricbeat/docs/metricbeat-options.asciidoc @@ -240,3 +240,20 @@ and then use the value in an HTTP Authorization header. An optional base path to be used in HTTP URIs. If defined, Metricbeat will insert this value as the first segment in the HTTP URI path. + +[float] +==== `query` + +An optional value to pass common query params in YAML. Instead of setting the query params +within hosts values using the syntax `?key=value&key2&value2`, you can set it here like this: + +[source,yaml] +---- +query: + key: value + key2: value2 + list: + - 1.1 + - 2.95 + - -15 +---- diff --git a/metricbeat/mb/mb.go b/metricbeat/mb/mb.go index e19b3fb108c..b9617274d22 100644 --- a/metricbeat/mb/mb.go +++ b/metricbeat/mb/mb.go @@ -23,6 +23,7 @@ package mb import ( "fmt" + "net/url" "time" "github.com/elastic/beats/libbeat/common" @@ -306,17 +307,46 @@ type ModuleConfig struct { MetricSets []string `config:"metricsets"` Enabled bool `config:"enabled"` Raw bool `config:"raw"` + Query QueryParams `config:"query"` } func (c ModuleConfig) String() string { return fmt.Sprintf(`{Module:"%v", MetricSets:%v, Enabled:%v, `+ - `Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v}`, + `Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v, Query:%v}`, c.Module, c.MetricSets, c.Enabled, len(c.Hosts), c.Period, c.Timeout, - c.Raw) + c.Raw, c.Query) } func (c ModuleConfig) GoString() string { return c.String() } +// QueryParams is a convenient map[string]interface{} wrapper to implement the String interface which returns the +// values in common query params format (key=value&key2=value2) which is the way that the url package expects this +// params (without the initial '?') +type QueryParams map[string]interface{} + +// String returns the values in common query params format (key=value&key2=value2) which is the way that the url +// package expects this params (without the initial '?') +func (q QueryParams) String() (s string) { + u := url.Values{} + + for k, v := range q { + if values, ok := v.([]interface{}); ok { + for _, innerValue := range values { + u.Add(k, fmt.Sprintf("%v", innerValue)) + } + } else { + //nil values in YAML shouldn't be stringified anyhow + if v == nil { + u.Add(k, "") + } else { + u.Add(k, fmt.Sprintf("%v", v)) + } + } + } + + return u.Encode() +} + // defaultModuleConfig contains the default values for ModuleConfig instances. var defaultModuleConfig = ModuleConfig{ Enabled: true, diff --git a/metricbeat/mb/mb_test.go b/metricbeat/mb/mb_test.go index c2b7ccfd36b..3e3b21ea30a 100644 --- a/metricbeat/mb/mb_test.go +++ b/metricbeat/mb/mb_test.go @@ -206,7 +206,7 @@ func TestNewModulesHostParser(t *testing.T) { r := newTestRegistry(t) factory := func(base BaseMetricSet) (MetricSet, error) { - return &testMetricSet{base}, nil + return &testMetricSet{BaseMetricSet: base}, nil } hostParser := func(m Module, rawHost string) (HostData, error) { @@ -375,3 +375,26 @@ func newConfig(t testing.TB, moduleConfig interface{}) *common.Config { } return config } + +func TestModuleConfigQueryParams(t *testing.T) { + qp := QueryParams{ + "stringKey": "value", + "intKey": 10, + "floatKey": 11.5, + "boolKey": true, + "nullKey": nil, + "arKey": []interface{}{1, 2}, + } + + res := qp.String() + + expectedValues := []string{"stringKey=value", "intKey=10", "floatKey=11.5", "boolKey=true", "nullKey=", "arKey=1", "arKey=2"} + for _, expected := range expectedValues { + assert.Contains(t, res, expected) + } + + assert.NotContains(t, res, "?") + assert.NotContains(t, res, "%") + assert.NotEqual(t, "&", res[0]) + assert.NotEqual(t, "&", res[len(res)-1]) +} diff --git a/metricbeat/mb/parse/url.go b/metricbeat/mb/parse/url.go index e8945d9df1f..66c970cbd6b 100644 --- a/metricbeat/mb/parse/url.go +++ b/metricbeat/mb/parse/url.go @@ -50,6 +50,16 @@ func (b URLHostParserBuilder) Build() mb.HostParser { return mb.HostData{}, err } + query, ok := conf["query"] + if ok { + queryMap, ok := query.(map[string]interface{}) + if !ok { + return mb.HostData{}, errors.Errorf("'query' config for module %v is not a map", module.Name()) + } + + b.QueryParams = mb.QueryParams(queryMap).String() + } + var user, pass, path, basePath string t, ok := conf["username"] if ok { diff --git a/metricbeat/mb/parse/url_test.go b/metricbeat/mb/parse/url_test.go index 571a5c0d13a..8465f4f1679 100644 --- a/metricbeat/mb/parse/url_test.go +++ b/metricbeat/mb/parse/url_test.go @@ -20,6 +20,8 @@ package parse import ( "testing" + "github.com/elastic/beats/metricbeat/mb" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/stretchr/testify/assert" @@ -118,6 +120,7 @@ func TestURLHostParserBuilder(t *testing.T) { {map[string]interface{}{"basepath": "foo/"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"}, {map[string]interface{}{"basepath": "/foo/"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"}, {map[string]interface{}{"basepath": "foo"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"}, + {map[string]interface{}{"basepath": "foo"}, URLHostParserBuilder{DefaultPath: "/queryParams", QueryParams: mb.QueryParams{"key": "value"}.String()}, "http://example.com/foo/queryParams?key=value"}, } for _, test := range cases {