Add regions parameter in aws module config

CHANGELOG.next.asciidoc

@@ -175,6 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `path_name` and `start_name` to service metricset on windows module {issue}8364[8364] {pull}11877[11877]
 - Add check on object name in the counter path if the instance name is missing {issue}6528[6528] {pull}11878[11878]
 - Add AWS cloudwatch metricset. {pull}11798[11798] {issue}11734[11734]
+- Add `regions` in aws module config to specify target regions for querying cloudwatch metrics. {issue}11932[11932] {pull}11956[11956]
 
 *Packetbeat*
 

metricbeat/docs/modules/aws.asciidoc

@@ -14,17 +14,21 @@ The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `c
 [float]
 === Module-specific configuration notes
 
-This module environment variable `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_REGION`
-references in the config file to set values that need to be configurable during deployment.
+This module uses environment variable `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_REGION`
+that are referenced in the config file to set values that need to be configurable during deployment. It also
+accept optional configuration `regions` to specify what are the AWS regions to query metrics from. If `regions`
+parameter is not set in the config file, then by default, aws module will query metrics from all available
+AWS regions.
 
 There are two different kinds of AWS credentials can be used here: `access keys` and `temporary security credentials`.
 `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are the two parts of `access keys`. They are long-term credentials for
 an IAM user or the AWS account root user. Please see
 https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[AWS Access Keys
  and Secret Access Keys] for more details. `temporary security credentials` has a limited lifetime and consists of an access key ID,
 a secret access key, and a security token which typically returned from `GetSessionToken`. MFA-enabled IAM users would
-need to submit an MFA code while calling `GetSessionToken`. `aws_default_region` is to set the region for SDK to use. Please
-see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials] for more details.
+need to submit an MFA code while calling `GetSessionToken`. `default_region` identifies the AWS Region whose servers you want to send
+your first API request to by default. This is typically the Region closest to you, but it can be any Region.
+Please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials] for more details.
 `sts get-session-token` AWS CLI can be used to generate temporary credentials. For example. with MFA-enabled:
 
 ----
@@ -56,6 +60,17 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY}'
   session_token: '${AWS_SESSION_TOKEN}'
   default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
+    - s3_daily_storage
+  access_key_id: '${AWS_ACCESS_KEY_ID}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY}'
+  session_token: '${AWS_SESSION_TOKEN}'
+  default_region: '${AWS_REGION:us-west-1}'
+  regions:
+    - us-west-1
+    - us-east-1
 ----
 
 [float]
@@ -121,11 +136,20 @@ metricbeat.modules:
   period: 300s
   metricsets:
     - ec2
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
     - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 86400s
   metricsets:
@@ -135,6 +159,8 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 300s
   metricsets:
@@ -150,6 +176,9 @@ metricbeat.modules:
         - name: InstanceId
           value: i-0686946e22cf9494a
     - namespace: AWS/EBS
+  #regions:
+  #  - us-east-1
+  #  - us-east-2
 ----
 
 [float]

x-pack/metricbeat/metricbeat.reference.yml

@@ -161,11 +161,20 @@ metricbeat.modules:
   period: 300s
   metricsets:
     - ec2
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
     - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 86400s
   metricsets:
@@ -175,6 +184,8 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 300s
   metricsets:
@@ -190,6 +201,9 @@ metricbeat.modules:
         - name: InstanceId
           value: i-0686946e22cf9494a
     - namespace: AWS/EBS
+  #regions:
+  #  - us-east-1
+  #  - us-east-2
 
 #--------------------------------- Ceph Module ---------------------------------
 - module: ceph

x-pack/metricbeat/module/aws/_meta/config.yml

@@ -2,11 +2,20 @@
   period: 300s
   metricsets:
     - ec2
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
     - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 86400s
   metricsets:
@@ -16,6 +25,8 @@
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 300s
   metricsets:
@@ -31,3 +42,6 @@
         - name: InstanceId
           value: i-0686946e22cf9494a
     - namespace: AWS/EBS
+  #regions:
+  #  - us-east-1
+  #  - us-east-2

x-pack/metricbeat/module/aws/_meta/docs.asciidoc

@@ -7,17 +7,21 @@ The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `c
 [float]
 === Module-specific configuration notes
 
-This module environment variable `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_REGION`
-references in the config file to set values that need to be configurable during deployment.
+This module uses environment variable `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` and `AWS_REGION`
+that are referenced in the config file to set values that need to be configurable during deployment. It also
+accept optional configuration `regions` to specify what are the AWS regions to query metrics from. If `regions`
+parameter is not set in the config file, then by default, aws module will query metrics from all available
+AWS regions.
 
 There are two different kinds of AWS credentials can be used here: `access keys` and `temporary security credentials`.
 `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are the two parts of `access keys`. They are long-term credentials for
 an IAM user or the AWS account root user. Please see
 https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[AWS Access Keys
  and Secret Access Keys] for more details. `temporary security credentials` has a limited lifetime and consists of an access key ID,
 a secret access key, and a security token which typically returned from `GetSessionToken`. MFA-enabled IAM users would
-need to submit an MFA code while calling `GetSessionToken`. `aws_default_region` is to set the region for SDK to use. Please
-see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials] for more details.
+need to submit an MFA code while calling `GetSessionToken`. `default_region` identifies the AWS Region whose servers you want to send
+your first API request to by default. This is typically the Region closest to you, but it can be any Region.
+Please see https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html[Temporary Security Credentials] for more details.
 `sts get-session-token` AWS CLI can be used to generate temporary credentials. For example. with MFA-enabled:
 
 ----
@@ -49,6 +53,17 @@ metricbeat.modules:
   secret_access_key: '${AWS_SECRET_ACCESS_KEY}'
   session_token: '${AWS_SESSION_TOKEN}'
   default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
+    - s3_daily_storage
+  access_key_id: '${AWS_ACCESS_KEY_ID}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY}'
+  session_token: '${AWS_SESSION_TOKEN}'
+  default_region: '${AWS_REGION:us-west-1}'
+  regions:
+    - us-west-1
+    - us-east-1
 ----
 
 [float]

x-pack/metricbeat/module/aws/aws.go

@@ -21,10 +21,11 @@ import (
 // Config defines all required and optional parameters for aws metricsets
 type Config struct {
 	Period          time.Duration `config:"period" validate:"nonzero,required"`
-	AccessKeyID     string        `config:"access_key_id"`
-	SecretAccessKey string        `config:"secret_access_key"`
+	AccessKeyID     string        `config:"access_key_id" validate:"nonzero,required"`
+	SecretAccessKey string        `config:"secret_access_key" validate:"nonzero,required"`
 	SessionToken    string        `config:"session_token"`
 	DefaultRegion   string        `config:"default_region"`
+	Regions         []string      `config:"regions"`
 }
 
 // MetricSet is the base metricset for all aws metricsets
@@ -62,43 +63,50 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 	}
 
 	awsConfig := defaults.Config()
-	awsCreds := awssdk.Credentials{
+	awsCredentials := awssdk.Credentials{
 		AccessKeyID:     config.AccessKeyID,
 		SecretAccessKey: config.SecretAccessKey,
 	}
 	if config.SessionToken != "" {
-		awsCreds.SessionToken = config.SessionToken
+		awsCredentials.SessionToken = config.SessionToken
 	}
 
 	awsConfig.Credentials = awssdk.StaticCredentialsProvider{
-		Value: awsCreds,
+		Value: awsCredentials,
 	}
 
 	awsConfig.Region = config.DefaultRegion
 
-	svcEC2 := ec2.New(awsConfig)
-	regionsList, err := getRegions(svcEC2)
-	if err != nil {
-		return nil, err
-	}
-
 	durationString, periodSec := convertPeriodToDuration(config.Period)
 	if err != nil {
 		return nil, err
 	}
 
-	// Construct MetricSet
 	metricSet := MetricSet{
 		BaseMetricSet:  base,
-		RegionsList:    regionsList,
 		DurationString: durationString,
 		PeriodInSec:    periodSec,
 		AwsConfig:      &awsConfig,
 	}
+
+	// Construct MetricSet with a full regions list
+	if config.Regions == nil {
+		svcEC2 := ec2.New(awsConfig)
+		completeRegionsList, err := getRegions(svcEC2)
+		if err != nil {
+			return nil, err
+		}
+
+		metricSet.RegionsList = completeRegionsList
+		return &metricSet, nil
+	}
+
+	// Construct MetricSet with specific regions list from config
+	metricSet.RegionsList = config.Regions
 	return &metricSet, nil
 }
 
-func getRegions(svc ec2iface.EC2API) (regionsList []string, err error) {
+func getRegions(svc ec2iface.EC2API) (completeRegionsList []string, err error) {
 	input := &ec2.DescribeRegionsInput{}
 	req := svc.DescribeRegionsRequest(input)
 	output, err := req.Send()
@@ -107,7 +115,7 @@ func getRegions(svc ec2iface.EC2API) (regionsList []string, err error) {
 		return
 	}
 	for _, region := range output.Regions {
-		regionsList = append(regionsList, *region.RegionName)
+		completeRegionsList = append(completeRegionsList, *region.RegionName)
 	}
 	return
 }

x-pack/metricbeat/module/aws/ec2/ec2.go

@@ -263,7 +263,7 @@ func getInstancesPerRegion(svc ec2iface.EC2API) (instanceIDs []string, instances
 func createMetricDataQuery(metric cloudwatch.Metric, instanceID string, index int, periodInSec int) (metricDataQuery cloudwatch.MetricDataQuery) {
 	statistic := "Average"
 	period := int64(periodInSec)
-	id := "ec2" + strconv.Itoa(index)
+	id := metricsetName + strconv.Itoa(index)
 	metricDims := metric.Dimensions
 
 	for _, dim := range metricDims {

x-pack/metricbeat/modules.d/aws.yml.disabled

@@ -2,11 +2,20 @@
   period: 300s
   metricsets:
     - ec2
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
     - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 86400s
   metricsets:
@@ -16,6 +25,8 @@
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+  #regions:
+  #  - us-west-1
 - module: aws
   period: 300s
   metricsets:
@@ -31,3 +42,6 @@
         - name: InstanceId
           value: i-0686946e22cf9494a
     - namespace: AWS/EBS
+  #regions:
+  #  - us-east-1
+  #  - us-east-2