From f940c36884d3749901a9c99bea5463a6030cdd9c Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Fri, 27 Sep 2019 02:23:20 +0200 Subject: [PATCH] =?UTF-8?q?Cherry-pick=20#13812=20to=207.4:=20Allow=20user?= =?UTF-8?q?s=20to=20select=20the=20cloud=20met=E2=80=A6=20(#13815)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Allow users to select the cloud metadata providers (#13812) * Allow users to select the cloud metadata providers We introduce a new setting 'providers' to the add_cloud_metadata processor. By now all the implementation for metadata providers requires developers to mark a provider as 'local'. The alibaba and tencent providers are not marked as local by now. If the 'providers' setting is not used, then no all providers marked as 'local' are applied. This is a breaking change, because alibaba and tencent providers will not be enabled anymore by default. If the providers setting is used, only the selected providers will be used. (cherry picked from commit dc997737e9aa639bd8e257e0390b0c485bb9006b) --- CHANGELOG.next.asciidoc | 2 + libbeat/docs/processors-using.asciidoc | 25 +- .../add_cloud_metadata/add_cloud_metadata.go | 314 ++---------------- .../processors/add_cloud_metadata/config.go | 76 +++++ .../add_cloud_metadata/http_fetcher.go | 159 +++++++++ .../provider_alibaba_cloud.go | 70 ++-- .../provider_alibaba_cloud_test.go | 3 +- .../add_cloud_metadata/provider_aws_ec2.go | 34 +- .../add_cloud_metadata/provider_azure_vm.go | 42 ++- .../provider_digital_ocean.go | 32 +- .../add_cloud_metadata/provider_google_gce.go | 84 ++--- .../provider_openstack_nova.go | 71 ++-- .../provider_tencent_cloud.go | 70 ++-- .../provider_tencent_cloud_test.go | 3 +- .../add_cloud_metadata/providers.go | 175 ++++++++++ .../add_cloud_metadata/providers_test.go | 100 ++++++ 16 files changed, 786 insertions(+), 474 deletions(-) create mode 100644 libbeat/processors/add_cloud_metadata/config.go create mode 100644 libbeat/processors/add_cloud_metadata/http_fetcher.go create mode 100644 libbeat/processors/add_cloud_metadata/providers.go create mode 100644 libbeat/processors/add_cloud_metadata/providers_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0ba2c71cc0a..72dffe9a70a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -11,6 +11,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* - Update to Golang 1.12.1. {pull}11330[11330] +- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812] *Auditbeat* @@ -71,6 +72,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Decouple Debug logging from fail_on_error logic for rename, copy, truncate processors {pull}12451[12451] - Allow a beat to ship monitoring data directly to an Elasticsearch monitoring cluster. {pull}9260[9260] +- Add `providers` setting to `add_cloud_metadata` processor. {pull}13812[13812] *Auditbeat* diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index ad48fe6a533..12807d04c75 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -505,8 +505,8 @@ not: === Add cloud metadata The `add_cloud_metadata` processor enriches each event with instance metadata -from the machine's hosting provider. At startup it will detect the hosting -provider and cache the instance metadata. +from the machine's hosting provider. At startup it will query a list of hosting +providers and cache the instance metadata. The following cloud providers are supported: @@ -518,6 +518,10 @@ The following cloud providers are supported: - Azure Virtual Machine - Openstack Nova +The Alibaba Cloud and Tencent cloud providers are disabled by default, because +they require to access a remote host. The `providers` setting allows users to +select a list of default providers to query. + The simple configuration below enables the processor. [source,yaml] @@ -526,7 +530,7 @@ processors: - add_cloud_metadata: ~ ------------------------------------------------------------------------------- -The `add_cloud_metadata` processor has two optional configuration settings. +The `add_cloud_metadata` processor has three optional configuration settings. The first one is `timeout` which specifies the maximum amount of time to wait for a successful response when detecting the hosting provider. The default timeout value is `3s`. @@ -535,7 +539,20 @@ If a timeout occurs then no instance metadata will be added to the events. This makes it possible to enable this processor for all your deployments (in the cloud or on-premise). -The second optional configuration setting is `overwrite`. When `overwrite` is +The second optional setting is `providers`. The `providers` settings accepts a +list of cloud provider names to be used. If `providers` is not configured, then +all providers that do not access a remote endpoint are enabled by default. + +List of names the `providers` setting supports: +- "alibaba", or "ecs" for the Alibaba Cloud provider (disabled by default). +- "azure" for Azure Virtual Machine (enabled by default). +- "digitalocean" for Digital Ocean (enabled by default). +- "aws", or "ec2" for Amazon Web Services (enabled by default). +- "gcp" for Google Copmute Enging (enabled by default). +- "openstack", or "nova" for Openstack Nova (enabled by default). +- "tencent", or "qcloud" for Tencent Cloud (disabled by default). + +The third optional configuration setting is `overwrite`. When `overwrite` is `true`, `add_cloud_metadata` overwrites existing `cloud.*` fields (`false` by default). diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index d61992457e4..1a7c01ce05b 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -18,13 +18,7 @@ package add_cloud_metadata import ( - "bytes" - "context" - "encoding/json" "fmt" - "io/ioutil" - "net" - "net/http" "sync" "time" @@ -40,12 +34,6 @@ const ( // metadataHost is the IP that each of the cloud providers supported here // use for their metadata service. metadataHost = "169.254.169.254" - - // Default config - defaultTimeOut = 3 * time.Second - - // Default overwrite - defaultOverwrite = false ) var debugf = logp.MakeDebug("filters") @@ -55,305 +43,58 @@ func init() { processors.RegisterPlugin("add_cloud_metadata", New) } -type schemaConv func(m map[string]interface{}) common.MapStr - -// responseHandler is the callback function that used to write something -// to the result according the HTTP response. -type responseHandler func(all []byte, res *result) error - -type metadataFetcher struct { - provider string - headers map[string]string - responseHandlers map[string]responseHandler - conv schemaConv -} - -// fetchRaw queries raw metadata from a hosting provider's metadata service. -func (f *metadataFetcher) fetchRaw( - ctx context.Context, - client http.Client, - url string, - responseHandler responseHandler, - result *result, -) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - result.err = errors.Wrapf(err, "failed to create http request for %v", f.provider) - return - } - for k, v := range f.headers { - req.Header.Add(k, v) - } - req = req.WithContext(ctx) - - rsp, err := client.Do(req) - if err != nil { - result.err = errors.Wrapf(err, "failed requesting %v metadata", f.provider) - return - } - defer rsp.Body.Close() - - if rsp.StatusCode != http.StatusOK { - result.err = errors.Errorf("failed with http status code %v", rsp.StatusCode) - return - } - - all, err := ioutil.ReadAll(rsp.Body) - if err != nil { - result.err = errors.Wrapf(err, "failed requesting %v metadata", f.provider) - return - } - - // Decode JSON. - err = responseHandler(all, result) - if err != nil { - result.err = err - return - } - - return -} - -// fetchMetadata queries metadata from a hosting provider's metadata service. -// Some providers require multiple HTTP requests to gather the whole metadata, -// len(f.responseHandlers) > 1 indicates that multiple requests are needed. -func (f *metadataFetcher) fetchMetadata(ctx context.Context, client http.Client) result { - res := result{provider: f.provider, metadata: common.MapStr{}} - for url, responseHandler := range f.responseHandlers { - f.fetchRaw(ctx, client, url, responseHandler, &res) - if res.err != nil { - return res - } - } - - // Apply schema. - res.metadata = f.conv(res.metadata) - res.metadata["provider"] = f.provider - - return res -} - -// result is the result of a query for a specific hosting provider's metadata. -type result struct { - provider string // Hosting provider type. - err error // Error that occurred while fetching (if any). - metadata common.MapStr // A specific subset of the metadata received from the hosting provider. -} - -func (r result) String() string { - return fmt.Sprintf("result=[provider:%v, error=%v, metadata=%v]", - r.provider, r.err, r.metadata) -} - -// writeResult blocks until it can write the result r to the channel c or until -// the context times out. -func writeResult(ctx context.Context, c chan result, r result) error { - select { - case <-ctx.Done(): - return ctx.Err() - case c <- r: - return nil - } -} - -// fetchMetadata attempts to fetch metadata in parallel from each of the -// hosting providers supported by this processor. It wait for the results to -// be returned or for a timeout to occur then returns the results that -// completed in time. -func fetchMetadata(metadataFetchers []*metadataFetcher, timeout time.Duration) *result { - debugf("add_cloud_metadata: starting to fetch metadata, timeout=%v", timeout) - start := time.Now() - defer func() { - debugf("add_cloud_metadata: fetchMetadata ran for %v", time.Since(start)) - }() - - // Create HTTP client with our timeouts and keep-alive disabled. - client := http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - DisableKeepAlives: true, - DialContext: (&net.Dialer{ - Timeout: timeout, - KeepAlive: 0, - }).DialContext, - }, - } - - // Create context to enable explicit cancellation of the http requests. - ctx, cancel := context.WithTimeout(context.TODO(), timeout) - defer cancel() - - c := make(chan result) - for _, fetcher := range metadataFetchers { - go func(fetcher *metadataFetcher) { - writeResult(ctx, c, fetcher.fetchMetadata(ctx, client)) - }(fetcher) - } - - for i := 0; i < len(metadataFetchers); i++ { - select { - case result := <-c: - debugf("add_cloud_metadata: received disposition for %v after %v. %v", - result.provider, time.Since(start), result) - // Bail out on first success. - if result.err == nil && result.metadata != nil { - return &result - } - case <-ctx.Done(): - debugf("add_cloud_metadata: timed-out waiting for all responses") - return nil - } - } - - return nil -} - -// getMetadataURLs loads config and generates the metadata URLs. -func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string) ([]string, error) { - var urls []string - config := struct { - MetadataHostAndPort string `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only). - }{ - MetadataHostAndPort: defaultHost, - } - err := c.Unpack(&config) - if err != nil { - return urls, errors.Wrap(err, "failed to unpack add_cloud_metadata config") - } - for _, uri := range metadataURIs { - urls = append(urls, "http://"+config.MetadataHostAndPort+uri) - } - return urls, nil -} - -// makeJSONPicker returns a responseHandler function that unmarshals JSON -// from a hosting provider's HTTP response and writes it to the result. -func makeJSONPicker(provider string) responseHandler { - return func(all []byte, res *result) error { - dec := json.NewDecoder(bytes.NewReader(all)) - dec.UseNumber() - err := dec.Decode(&res.metadata) - if err != nil { - err = errors.Wrapf(err, "failed to unmarshal %v JSON of '%v'", provider, string(all)) - return err - } - return nil - } -} - -// newMetadataFetcher return metadataFetcher with one pass JSON responseHandler. -func newMetadataFetcher( - c *common.Config, - provider string, - headers map[string]string, - host string, - conv schemaConv, - uri string, -) (*metadataFetcher, error) { - urls, err := getMetadataURLs(c, host, []string{uri}) - if err != nil { - return nil, err - } - responseHandlers := map[string]responseHandler{urls[0]: makeJSONPicker(provider)} - fetcher := &metadataFetcher{provider, headers, responseHandlers, conv} - return fetcher, nil +type addCloudMetadata struct { + initOnce sync.Once + initData *initData + metadata common.MapStr } -func setupFetchers(c *common.Config) ([]*metadataFetcher, error) { - var fetchers []*metadataFetcher - doFetcher, err := newDoMetadataFetcher(c) - if err != nil { - return fetchers, err - } - ec2Fetcher, err := newEc2MetadataFetcher(c) - if err != nil { - return fetchers, err - } - gceFetcher, err := newGceMetadataFetcher(c) - if err != nil { - return fetchers, err - } - qcloudFetcher, err := newQcloudMetadataFetcher(c) - if err != nil { - return fetchers, err - } - ecsFetcher, err := newAlibabaCloudMetadataFetcher(c) - if err != nil { - return fetchers, err - } - azFetcher, err := newAzureVmMetadataFetcher(c) - if err != nil { - return fetchers, err - } - osFetcher, err := newOpenstackNovaMetadataFetcher(c) - if err != nil { - return fetchers, err - } - - fetchers = []*metadataFetcher{ - doFetcher, - ec2Fetcher, - gceFetcher, - qcloudFetcher, - ecsFetcher, - azFetcher, - osFetcher, - } - return fetchers, nil +type initData struct { + fetchers []metadataFetcher + timeout time.Duration + overwrite bool } // New constructs a new add_cloud_metadata processor. func New(c *common.Config) (processors.Processor, error) { - config := struct { - Timeout time.Duration `config:"timeout"` // Amount of time to wait for responses from the metadata services. - Overwrite bool `config:"overwrite"` // Overwrite if cloud.* fields already exist. - }{ - Timeout: defaultTimeOut, - Overwrite: defaultOverwrite, - } - err := c.Unpack(&config) - if err != nil { + config := defaultConfig() + if err := c.Unpack(&config); err != nil { return nil, errors.Wrap(err, "failed to unpack add_cloud_metadata config") } - fetchers, err := setupFetchers(c) + initProviders := selectProviders(config.Providers, cloudMetaProviders) + fetchers, err := setupFetchers(initProviders, c) if err != nil { return nil, err } - p := &addCloudMetadata{ initData: &initData{fetchers, config.Timeout, config.Overwrite}, } - go p.initOnce.Do(p.init) + go p.init() return p, nil } -type initData struct { - fetchers []*metadataFetcher - timeout time.Duration - overwrite bool -} - -type addCloudMetadata struct { - initOnce sync.Once - initData *initData - metadata common.MapStr +func (r result) String() string { + return fmt.Sprintf("result=[provider:%v, error=%v, metadata=%v]", + r.provider, r.err, r.metadata) } func (p *addCloudMetadata) init() { - result := fetchMetadata(p.initData.fetchers, p.initData.timeout) - if result == nil { - logp.Info("add_cloud_metadata: hosting provider type not detected.") - return - } - p.metadata = result.metadata - logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v", - result.provider, result.metadata.String()) + p.initOnce.Do(func() { + result := fetchMetadata(p.initData.fetchers, p.initData.timeout) + if result == nil { + logp.Info("add_cloud_metadata: hosting provider type not detected.") + return + } + p.metadata = result.metadata + logp.Info("add_cloud_metadata: hosting provider type detected as %v, metadata=%v", + result.provider, result.metadata.String()) + }) } func (p *addCloudMetadata) getMeta() common.MapStr { - p.initOnce.Do(p.init) + p.init() return p.metadata } @@ -374,7 +115,6 @@ func (p *addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) { } _, err := event.PutValue("cloud", meta) - return event, err } diff --git a/libbeat/processors/add_cloud_metadata/config.go b/libbeat/processors/add_cloud_metadata/config.go new file mode 100644 index 00000000000..08f4a241483 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/config.go @@ -0,0 +1,76 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package add_cloud_metadata + +import ( + "fmt" + "time" +) + +type config struct { + Timeout time.Duration `config:"timeout"` // Amount of time to wait for responses from the metadata services. + Overwrite bool `config:"overwrite"` // Overwrite if cloud.* fields already exist. + Providers providerList `config:"providers"` // List of providers to probe +} + +type providerList []string + +const ( + // Default config + defaultTimeout = 3 * time.Second + + // Default overwrite + defaultOverwrite = false +) + +func defaultConfig() config { + return config{ + Timeout: defaultTimeout, + Overwrite: defaultOverwrite, + Providers: nil, // enable all local-only providers by default + } +} + +func (c *config) Validate() error { + // XXX: remove this check. A bug in go-ucfg prevents the correct validation + // on providerList + return c.Providers.Validate() +} + +func (l providerList) Has(name string) bool { + for _, elem := range l { + if string(elem) == name { + return true + } + } + return false +} + +func (l *providerList) Validate() error { + if l == nil { + return nil + } + + for _, name := range *l { + if _, ok := cloudMetaProviders[name]; !ok { + return fmt.Errorf("unknown provider '%v'", name) + } + } + return nil + +} diff --git a/libbeat/processors/add_cloud_metadata/http_fetcher.go b/libbeat/processors/add_cloud_metadata/http_fetcher.go new file mode 100644 index 00000000000..30e31244333 --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/http_fetcher.go @@ -0,0 +1,159 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package add_cloud_metadata + +import ( + "bytes" + "context" + "encoding/json" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" +) + +type httpMetadataFetcher struct { + provider string + headers map[string]string + responseHandlers map[string]responseHandler + conv schemaConv +} + +// responseHandler is the callback function that used to write something +// to the result according the HTTP response. +type responseHandler func(all []byte, res *result) error + +type schemaConv func(m map[string]interface{}) common.MapStr + +// newMetadataFetcher return metadataFetcher with one pass JSON responseHandler. +func newMetadataFetcher( + c *common.Config, + provider string, + headers map[string]string, + host string, + conv schemaConv, + uri string, +) (*httpMetadataFetcher, error) { + urls, err := getMetadataURLs(c, host, []string{uri}) + if err != nil { + return nil, err + } + responseHandlers := map[string]responseHandler{urls[0]: makeJSONPicker(provider)} + fetcher := &httpMetadataFetcher{provider, headers, responseHandlers, conv} + return fetcher, nil +} + +// fetchMetadata queries metadata from a hosting provider's metadata service. +// Some providers require multiple HTTP requests to gather the whole metadata, +// len(f.responseHandlers) > 1 indicates that multiple requests are needed. +func (f *httpMetadataFetcher) fetchMetadata(ctx context.Context, client http.Client) result { + res := result{provider: f.provider, metadata: common.MapStr{}} + for url, responseHandler := range f.responseHandlers { + f.fetchRaw(ctx, client, url, responseHandler, &res) + if res.err != nil { + return res + } + } + + // Apply schema. + res.metadata = f.conv(res.metadata) + res.metadata["provider"] = f.provider + + return res +} + +// fetchRaw queries raw metadata from a hosting provider's metadata service. +func (f *httpMetadataFetcher) fetchRaw( + ctx context.Context, + client http.Client, + url string, + responseHandler responseHandler, + result *result, +) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + result.err = errors.Wrapf(err, "failed to create http request for %v", f.provider) + return + } + for k, v := range f.headers { + req.Header.Add(k, v) + } + req = req.WithContext(ctx) + + rsp, err := client.Do(req) + if err != nil { + result.err = errors.Wrapf(err, "failed requesting %v metadata", f.provider) + return + } + defer rsp.Body.Close() + + if rsp.StatusCode != http.StatusOK { + result.err = errors.Errorf("failed with http status code %v", rsp.StatusCode) + return + } + + all, err := ioutil.ReadAll(rsp.Body) + if err != nil { + result.err = errors.Wrapf(err, "failed requesting %v metadata", f.provider) + return + } + + // Decode JSON. + err = responseHandler(all, result) + if err != nil { + result.err = err + return + } + + return +} + +// getMetadataURLs loads config and generates the metadata URLs. +func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string) ([]string, error) { + var urls []string + config := struct { + MetadataHostAndPort string `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only). + }{ + MetadataHostAndPort: defaultHost, + } + err := c.Unpack(&config) + if err != nil { + return urls, errors.Wrap(err, "failed to unpack add_cloud_metadata config") + } + for _, uri := range metadataURIs { + urls = append(urls, "http://"+config.MetadataHostAndPort+uri) + } + return urls, nil +} + +// makeJSONPicker returns a responseHandler function that unmarshals JSON +// from a hosting provider's HTTP response and writes it to the result. +func makeJSONPicker(provider string) responseHandler { + return func(all []byte, res *result) error { + dec := json.NewDecoder(bytes.NewReader(all)) + dec.UseNumber() + err := dec.Decode(&res.metadata) + if err != nil { + err = errors.Wrapf(err, "failed to unmarshal %v JSON of '%v'", provider, string(all)) + return err + } + return nil + } +} diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go index 02fc158d405..afa502990a2 100644 --- a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go @@ -21,38 +21,44 @@ import "github.com/elastic/beats/libbeat/common" // Alibaba Cloud Metadata Service // Document https://help.aliyun.com/knowledge_detail/49122.html -func newAlibabaCloudMetadataFetcher(c *common.Config) (*metadataFetcher, error) { - ecsMetadataHost := "100.100.100.200" - ecsMetadataInstanceIDURI := "/latest/meta-data/instance-id" - ecsMetadataRegionURI := "/latest/meta-data/region-id" - ecsMetadataZoneURI := "/latest/meta-data/zone-id" +var alibabaCloudMetadataFetcher = provider{ + Name: "alibaba-ecs", - ecsSchema := func(m map[string]interface{}) common.MapStr { - return common.MapStr(m) - } + Local: false, - urls, err := getMetadataURLs(c, ecsMetadataHost, []string{ - ecsMetadataInstanceIDURI, - ecsMetadataRegionURI, - ecsMetadataZoneURI, - }) - if err != nil { - return nil, err - } - responseHandlers := map[string]responseHandler{ - urls[0]: func(all []byte, result *result) error { - result.metadata.Put("instance.id", string(all)) - return nil - }, - urls[1]: func(all []byte, result *result) error { - result.metadata["region"] = string(all) - return nil - }, - urls[2]: func(all []byte, result *result) error { - result.metadata["availability_zone"] = string(all) - return nil - }, - } - fetcher := &metadataFetcher{"ecs", nil, responseHandlers, ecsSchema} - return fetcher, nil + Create: func(_ string, c *common.Config) (metadataFetcher, error) { + ecsMetadataHost := "100.100.100.200" + ecsMetadataInstanceIDURI := "/latest/meta-data/instance-id" + ecsMetadataRegionURI := "/latest/meta-data/region-id" + ecsMetadataZoneURI := "/latest/meta-data/zone-id" + + ecsSchema := func(m map[string]interface{}) common.MapStr { + return common.MapStr(m) + } + + urls, err := getMetadataURLs(c, ecsMetadataHost, []string{ + ecsMetadataInstanceIDURI, + ecsMetadataRegionURI, + ecsMetadataZoneURI, + }) + if err != nil { + return nil, err + } + responseHandlers := map[string]responseHandler{ + urls[0]: func(all []byte, result *result) error { + result.metadata.Put("instance.id", string(all)) + return nil + }, + urls[1]: func(all []byte, result *result) error { + result.metadata["region"] = string(all) + return nil + }, + urls[2]: func(all []byte, result *result) error { + result.metadata["availability_zone"] = string(all) + return nil + }, + } + fetcher := &httpMetadataFetcher{"ecs", nil, responseHandlers, ecsSchema} + return fetcher, nil + }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go index a35fb5c3de3..7a6b7265d69 100644 --- a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud_test.go @@ -55,7 +55,8 @@ func TestRetrieveAlibabaCloudMetadata(t *testing.T) { defer server.Close() config, err := common.NewConfigFrom(map[string]interface{}{ - "host": server.Listener.Addr().String(), + "providers": []string{"alibaba"}, + "host": server.Listener.Addr().String(), }) if err != nil { diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go index 77b7869334f..ed28f3b38aa 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go @@ -26,19 +26,25 @@ import ( const ec2InstanceIdentityURI = "/2014-02-25/dynamic/instance-identity/document" // AWS EC2 Metadata Service -func newEc2MetadataFetcher(config *common.Config) (*metadataFetcher, error) { - ec2Schema := func(m map[string]interface{}) common.MapStr { - out, _ := s.Schema{ - "instance": s.Object{"id": c.Str("instanceId")}, - "machine": s.Object{"type": c.Str("instanceType")}, - "region": c.Str("region"), - "availability_zone": c.Str("availabilityZone"), - "account": s.Object{"id": c.Str("accountId")}, - "image": s.Object{"id": c.Str("imageId")}, - }.Apply(m) - return out - } +var ec2MetadataFetcher = provider{ + Name: "aws-ec2", - fetcher, err := newMetadataFetcher(config, "aws", nil, metadataHost, ec2Schema, ec2InstanceIdentityURI) - return fetcher, err + Local: true, + + Create: func(_ string, config *common.Config) (metadataFetcher, error) { + ec2Schema := func(m map[string]interface{}) common.MapStr { + out, _ := s.Schema{ + "instance": s.Object{"id": c.Str("instanceId")}, + "machine": s.Object{"type": c.Str("instanceType")}, + "region": c.Str("region"), + "availability_zone": c.Str("availabilityZone"), + "account": s.Object{"id": c.Str("accountId")}, + "image": s.Object{"id": c.Str("imageId")}, + }.Apply(m) + return out + } + + fetcher, err := newMetadataFetcher(config, "aws", nil, metadataHost, ec2Schema, ec2InstanceIdentityURI) + return fetcher, err + }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go index bbf5e19ff04..8ee84f417e3 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -24,23 +24,29 @@ import ( ) // Azure VM Metadata Service -func newAzureVmMetadataFetcher(config *common.Config) (*metadataFetcher, error) { - azMetadataURI := "/metadata/instance/compute?api-version=2017-04-02" - azHeaders := map[string]string{"Metadata": "true"} - azSchema := func(m map[string]interface{}) common.MapStr { - out, _ := s.Schema{ - "instance": s.Object{ - "id": c.Str("vmId"), - "name": c.Str("name"), - }, - "machine": s.Object{ - "type": c.Str("vmSize"), - }, - "region": c.Str("location"), - }.Apply(m) - return out - } +var azureVMMetadataFetcher = provider{ + Name: "azure-compute", - fetcher, err := newMetadataFetcher(config, "az", azHeaders, metadataHost, azSchema, azMetadataURI) - return fetcher, err + Local: true, + + Create: func(_ string, config *common.Config) (metadataFetcher, error) { + azMetadataURI := "/metadata/instance/compute?api-version=2017-04-02" + azHeaders := map[string]string{"Metadata": "true"} + azSchema := func(m map[string]interface{}) common.MapStr { + out, _ := s.Schema{ + "instance": s.Object{ + "id": c.Str("vmId"), + "name": c.Str("name"), + }, + "machine": s.Object{ + "type": c.Str("vmSize"), + }, + "region": c.Str("location"), + }.Apply(m) + return out + } + + fetcher, err := newMetadataFetcher(config, "az", azHeaders, metadataHost, azSchema, azMetadataURI) + return fetcher, err + }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go index d3500dd49bc..fe016ffb95e 100644 --- a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go @@ -24,18 +24,24 @@ import ( ) // DigitalOcean Metadata Service -func newDoMetadataFetcher(config *common.Config) (*metadataFetcher, error) { - doSchema := func(m map[string]interface{}) common.MapStr { - out, _ := s.Schema{ - "instance": s.Object{ - "id": c.StrFromNum("droplet_id"), - }, - "region": c.Str("region"), - }.Apply(m) - return out - } - doMetadataURI := "/metadata/v1.json" +var doMetadataFetcher = provider{ + Name: "digitalocean", - fetcher, err := newMetadataFetcher(config, "digitalocean", nil, metadataHost, doSchema, doMetadataURI) - return fetcher, err + Local: true, + + Create: func(provider string, config *common.Config) (metadataFetcher, error) { + doSchema := func(m map[string]interface{}) common.MapStr { + out, _ := s.Schema{ + "instance": s.Object{ + "id": c.StrFromNum("droplet_id"), + }, + "region": c.Str("region"), + }.Apply(m) + return out + } + doMetadataURI := "/metadata/v1.json" + + fetcher, err := newMetadataFetcher(config, provider, nil, metadataHost, doSchema, doMetadataURI) + return fetcher, err + }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce.go b/libbeat/processors/add_cloud_metadata/provider_google_gce.go index c69fb389531..27c574f9f1d 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce.go @@ -26,50 +26,56 @@ import ( ) // Google GCE Metadata Service -func newGceMetadataFetcher(config *common.Config) (*metadataFetcher, error) { - gceMetadataURI := "/computeMetadata/v1/?recursive=true&alt=json" - gceHeaders := map[string]string{"Metadata-Flavor": "Google"} - gceSchema := func(m map[string]interface{}) common.MapStr { - out := common.MapStr{} +var gceMetadataFetcher = provider{ + Name: "google-gce", - trimLeadingPath := func(key string) { - v, err := out.GetValue(key) - if err != nil { - return + Local: true, + + Create: func(provider string, config *common.Config) (metadataFetcher, error) { + gceMetadataURI := "/computeMetadata/v1/?recursive=true&alt=json" + gceHeaders := map[string]string{"Metadata-Flavor": "Google"} + gceSchema := func(m map[string]interface{}) common.MapStr { + out := common.MapStr{} + + trimLeadingPath := func(key string) { + v, err := out.GetValue(key) + if err != nil { + return + } + p, ok := v.(string) + if !ok { + return + } + out.Put(key, path.Base(p)) } - p, ok := v.(string) - if !ok { - return + + if instance, ok := m["instance"].(map[string]interface{}); ok { + s.Schema{ + "instance": s.Object{ + "id": c.StrFromNum("id"), + "name": c.Str("name"), + }, + "machine": s.Object{ + "type": c.Str("machineType"), + }, + "availability_zone": c.Str("zone"), + }.ApplyTo(out, instance) + trimLeadingPath("machine.type") + trimLeadingPath("availability_zone") } - out.Put(key, path.Base(p)) - } - if instance, ok := m["instance"].(map[string]interface{}); ok { - s.Schema{ - "instance": s.Object{ - "id": c.StrFromNum("id"), - "name": c.Str("name"), - }, - "machine": s.Object{ - "type": c.Str("machineType"), - }, - "availability_zone": c.Str("zone"), - }.ApplyTo(out, instance) - trimLeadingPath("machine.type") - trimLeadingPath("availability_zone") - } + if project, ok := m["project"].(map[string]interface{}); ok { + s.Schema{ + "project": s.Object{ + "id": c.Str("projectId"), + }, + }.ApplyTo(out, project) + } - if project, ok := m["project"].(map[string]interface{}); ok { - s.Schema{ - "project": s.Object{ - "id": c.Str("projectId"), - }, - }.ApplyTo(out, project) + return out } - return out - } - - fetcher, err := newMetadataFetcher(config, "gcp", gceHeaders, metadataHost, gceSchema, gceMetadataURI) - return fetcher, err + fetcher, err := newMetadataFetcher(config, provider, gceHeaders, metadataHost, gceSchema, gceMetadataURI) + return fetcher, err + }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go index d5b50560169..cbb516059b9 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go @@ -31,40 +31,45 @@ const ( // newOpenstackNovaMetadataFetcher returns a metadataFetcher for the // OpenStack Nova Metadata Service // Document https://docs.openstack.org/nova/latest/user/metadata-service.html -func newOpenstackNovaMetadataFetcher(c *common.Config) (*metadataFetcher, error) { +var openstackNovaMetadataFetcher = provider{ + Name: "openstack-nova", - osSchema := func(m map[string]interface{}) common.MapStr { - return common.MapStr(m) - } + Local: true, - urls, err := getMetadataURLs(c, metadataHost, []string{ - osMetadataInstanceIDURI, - osMetadataInstanceTypeURI, - osMetadataHostnameURI, - osMetadataZoneURI, - }) - if err != nil { - return nil, err - } + Create: func(provider string, c *common.Config) (metadataFetcher, error) { + osSchema := func(m map[string]interface{}) common.MapStr { + return common.MapStr(m) + } - responseHandlers := map[string]responseHandler{ - urls[0]: func(all []byte, result *result) error { - result.metadata.Put("instance.id", string(all)) - return nil - }, - urls[1]: func(all []byte, result *result) error { - result.metadata.Put("machine.type", string(all)) - return nil - }, - urls[2]: func(all []byte, result *result) error { - result.metadata.Put("instance.name", string(all)) - return nil - }, - urls[3]: func(all []byte, result *result) error { - result.metadata["availability_zone"] = string(all) - return nil - }, - } - fetcher := &metadataFetcher{"openstack", nil, responseHandlers, osSchema} - return fetcher, nil + urls, err := getMetadataURLs(c, metadataHost, []string{ + osMetadataInstanceIDURI, + osMetadataInstanceTypeURI, + osMetadataHostnameURI, + osMetadataZoneURI, + }) + if err != nil { + return nil, err + } + + responseHandlers := map[string]responseHandler{ + urls[0]: func(all []byte, result *result) error { + result.metadata.Put("instance.id", string(all)) + return nil + }, + urls[1]: func(all []byte, result *result) error { + result.metadata.Put("machine.type", string(all)) + return nil + }, + urls[2]: func(all []byte, result *result) error { + result.metadata.Put("instance.name", string(all)) + return nil + }, + urls[3]: func(all []byte, result *result) error { + result.metadata["availability_zone"] = string(all) + return nil + }, + } + fetcher := &httpMetadataFetcher{"openstack", nil, responseHandlers, osSchema} + return fetcher, nil + }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go index 6a380377de1..d24778c6850 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go @@ -21,38 +21,44 @@ import "github.com/elastic/beats/libbeat/common" // Tencent Cloud Metadata Service // Document https://www.qcloud.com/document/product/213/4934 -func newQcloudMetadataFetcher(c *common.Config) (*metadataFetcher, error) { - qcloudMetadataHost := "metadata.tencentyun.com" - qcloudMetadataInstanceIDURI := "/meta-data/instance-id" - qcloudMetadataRegionURI := "/meta-data/placement/region" - qcloudMetadataZoneURI := "/meta-data/placement/zone" +var qcloudMetadataFetcher = provider{ + Name: "tencent-qcloud", - qcloudSchema := func(m map[string]interface{}) common.MapStr { - return common.MapStr(m) - } + Local: false, - urls, err := getMetadataURLs(c, qcloudMetadataHost, []string{ - qcloudMetadataInstanceIDURI, - qcloudMetadataRegionURI, - qcloudMetadataZoneURI, - }) - if err != nil { - return nil, err - } - responseHandlers := map[string]responseHandler{ - urls[0]: func(all []byte, result *result) error { - result.metadata.Put("instance.id", string(all)) - return nil - }, - urls[1]: func(all []byte, result *result) error { - result.metadata["region"] = string(all) - return nil - }, - urls[2]: func(all []byte, result *result) error { - result.metadata["availability_zone"] = string(all) - return nil - }, - } - fetcher := &metadataFetcher{"qcloud", nil, responseHandlers, qcloudSchema} - return fetcher, nil + Create: func(_ string, c *common.Config) (metadataFetcher, error) { + qcloudMetadataHost := "metadata.tencentyun.com" + qcloudMetadataInstanceIDURI := "/meta-data/instance-id" + qcloudMetadataRegionURI := "/meta-data/placement/region" + qcloudMetadataZoneURI := "/meta-data/placement/zone" + + qcloudSchema := func(m map[string]interface{}) common.MapStr { + return common.MapStr(m) + } + + urls, err := getMetadataURLs(c, qcloudMetadataHost, []string{ + qcloudMetadataInstanceIDURI, + qcloudMetadataRegionURI, + qcloudMetadataZoneURI, + }) + if err != nil { + return nil, err + } + responseHandlers := map[string]responseHandler{ + urls[0]: func(all []byte, result *result) error { + result.metadata.Put("instance.id", string(all)) + return nil + }, + urls[1]: func(all []byte, result *result) error { + result.metadata["region"] = string(all) + return nil + }, + urls[2]: func(all []byte, result *result) error { + result.metadata["availability_zone"] = string(all) + return nil + }, + } + fetcher := &httpMetadataFetcher{"qcloud", nil, responseHandlers, qcloudSchema} + return fetcher, nil + }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go index c7868148ad1..72be1934c2d 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud_test.go @@ -55,7 +55,8 @@ func TestRetrieveQCloudMetadata(t *testing.T) { defer server.Close() config, err := common.NewConfigFrom(map[string]interface{}{ - "host": server.Listener.Addr().String(), + "providers": []string{"tencent"}, + "host": server.Listener.Addr().String(), }) if err != nil { diff --git a/libbeat/processors/add_cloud_metadata/providers.go b/libbeat/processors/add_cloud_metadata/providers.go new file mode 100644 index 00000000000..301e7d4731f --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/providers.go @@ -0,0 +1,175 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package add_cloud_metadata + +import ( + "context" + "net" + "net/http" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" +) + +type provider struct { + // Name contains a long name of provider and service metadata is fetched from. + Name string + + // Local Set to true if local IP is accessed only + Local bool + + // Create returns an actual metadataFetcher + Create func(string, *common.Config) (metadataFetcher, error) +} + +type metadataFetcher interface { + fetchMetadata(context.Context, http.Client) result +} + +// result is the result of a query for a specific hosting provider's metadata. +type result struct { + provider string // Hosting provider type. + err error // Error that occurred while fetching (if any). + metadata common.MapStr // A specific subset of the metadata received from the hosting provider. +} + +var cloudMetaProviders = map[string]provider{ + "alibaba": alibabaCloudMetadataFetcher, + "ecs": alibabaCloudMetadataFetcher, + "azure": azureVMMetadataFetcher, + "digitalocean": doMetadataFetcher, + "aws": ec2MetadataFetcher, + "ec2": ec2MetadataFetcher, + "gcp": gceMetadataFetcher, + "openstack": openstackNovaMetadataFetcher, + "nova": openstackNovaMetadataFetcher, + "qcloud": qcloudMetadataFetcher, + "tencent": qcloudMetadataFetcher, +} + +func selectProviders(configList providerList, providers map[string]provider) map[string]provider { + return filterMetaProviders(providersFilter(configList, providers), providers) +} + +func providersFilter(configList providerList, allProviders map[string]provider) func(string) bool { + if len(configList) == 0 { + return func(name string) bool { + ff, ok := allProviders[name] + return ok && ff.Local + } + } + return func(name string) (ok bool) { + if ok = configList.Has(name); ok { + _, ok = allProviders[name] + } + return ok + } +} + +func filterMetaProviders(filter func(string) bool, fetchers map[string]provider) map[string]provider { + out := map[string]provider{} + for name, ff := range fetchers { + if filter(name) { + out[name] = ff + } + } + return out +} + +func setupFetchers(providers map[string]provider, c *common.Config) ([]metadataFetcher, error) { + mf := make([]metadataFetcher, 0, len(providers)) + visited := map[string]bool{} + + // Iterate over all providers and create an unique meta-data fetcher per provider type. + // Some providers might appear twice in the set of providers to support aliases on provider names. + // For example aws and ec2 both use the same provider. + // The loop tracks already seen providers in the `visited` set, to ensure that we do not create + // duplicate providers for aliases. + for name, ff := range providers { + if visited[ff.Name] { + continue + } + visited[ff.Name] = true + + fetcher, err := ff.Create(name, c) + if err != nil { + return nil, errors.Wrapf(err, "failed to initialize the %v fetcher", name) + } + + mf = append(mf, fetcher) + } + return mf, nil +} + +// fetchMetadata attempts to fetch metadata in parallel from each of the +// hosting providers supported by this processor. It wait for the results to +// be returned or for a timeout to occur then returns the first result that +// completed in time. +func fetchMetadata(metadataFetchers []metadataFetcher, timeout time.Duration) *result { + debugf("add_cloud_metadata: starting to fetch metadata, timeout=%v", timeout) + start := time.Now() + defer func() { + debugf("add_cloud_metadata: fetchMetadata ran for %v", time.Since(start)) + }() + + // Create HTTP client with our timeouts and keep-alive disabled. + client := http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + DisableKeepAlives: true, + DialContext: (&net.Dialer{ + Timeout: timeout, + KeepAlive: 0, + }).DialContext, + }, + } + + // Create context to enable explicit cancellation of the http requests. + ctx, cancel := context.WithTimeout(context.TODO(), timeout) + defer cancel() + + results := make(chan result) + for _, fetcher := range metadataFetchers { + fetcher := fetcher + go func() { + select { + case <-ctx.Done(): + case results <- fetcher.fetchMetadata(ctx, client): + } + }() + } + + for i := 0; i < len(metadataFetchers); i++ { + select { + case result := <-results: + debugf("add_cloud_metadata: received disposition for %v after %v. %v", + result.provider, time.Since(start), result) + // Bail out on first success. + if result.err == nil && result.metadata != nil { + return &result + } + case <-ctx.Done(): + debugf("add_cloud_metadata: timed-out waiting for all responses") + return nil + } + } + + return nil +} diff --git a/libbeat/processors/add_cloud_metadata/providers_test.go b/libbeat/processors/add_cloud_metadata/providers_test.go new file mode 100644 index 00000000000..e9c8732c54a --- /dev/null +++ b/libbeat/processors/add_cloud_metadata/providers_test.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package add_cloud_metadata + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +func TestProvidersFilter(t *testing.T) { + var all []string + var allLocal []string + for name, ff := range cloudMetaProviders { + all = append(all, name) + if ff.Local { + allLocal = append(allLocal, name) + } + } + + cases := map[string]struct { + config map[string]interface{} + fail bool + expected []string + }{ + "all with local access only if not configured": { + config: map[string]interface{}{}, + expected: allLocal, + }, + "fail to load if unknown name is used": { + config: map[string]interface{}{ + "providers": []string{"unknown"}, + }, + fail: true, + }, + "only selected": { + config: map[string]interface{}{ + "providers": []string{"aws", "gcp", "digitalocean"}, + }, + }, + } + + copyStrings := func(in []string) (out []string) { + for _, str := range in { + out = append(out, str) + } + return out + } + + for name, test := range cases { + t.Run(name, func(t *testing.T) { + rawConfig := common.MustNewConfigFrom(test.config) + + config := defaultConfig() + err := rawConfig.Unpack(&config) + if err == nil && test.fail { + t.Fatal("Did expect to fail on unpack") + } else if err != nil && !test.fail { + t.Fatal("Unpack failed", err) + } else if test.fail && err != nil { + return + } + + // compute list of providers that should have matched + var expected []string + if len(test.expected) == 0 && len(config.Providers) > 0 { + expected = copyStrings(config.Providers) + } else { + expected = copyStrings(test.expected) + } + sort.Strings(expected) + + var actual []string + for name := range selectProviders(config.Providers, cloudMetaProviders) { + actual = append(actual, name) + } + + sort.Strings(actual) + assert.Equal(t, expected, actual) + }) + } +}