diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b77935ef89b..2ae485491db 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -842,6 +842,7 @@ same journal. {pull}18467[18467] - Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381] - Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703] - Duplicate system.process.cmdline field with process.command_line ECS field name. {pull}22325[22325] +- Add awsfargate module task_stats metricset to monitor AWS ECS Fargate. {pull}22034[22034] - Add connection and route metricsets for nats metricbeat module to collect metrics per connection/route. {pull}22445[22445] - Add unit file states to system/service {pull}22557[22557] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 5a706cea782..4fce6fb4a42 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -17,6 +17,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -4589,6 +4590,534 @@ type: double -- +[[exported-fields-awsfargate]] +== AWS Fargate fields + +`awsfargate` module collects AWS fargate metrics from task metadata endpoint. + + + +[float] +=== task_stats + +`task_stats` contains the metrics that were scraped from AWS fargate task stats ${ECS_CONTAINER_METADATA_URI_V4}/task/stats metadata endpoint. + + + +[float] +=== cpu + +Runtime CPU metrics. + + +*`task_stats.cpu.kernel.pct`*:: ++ +-- +Percentage of time in kernel space. + + +type: scaled_float + +format: percent + +-- + +*`task_stats.cpu.kernel.norm.pct`*:: ++ +-- +Percentage of time in kernel space normalized by the number of CPU cores. + + +type: scaled_float + +format: percent + +-- + +*`task_stats.cpu.kernel.ticks`*:: ++ +-- +CPU ticks in kernel space. + + +type: long + +-- + +*`task_stats.cpu.system.pct`*:: ++ +-- +Percentage of total CPU time in the system. + + +type: scaled_float + +format: percent + +-- + +*`task_stats.cpu.system.norm.pct`*:: ++ +-- +Percentage of total CPU time in the system normalized by the number of CPU cores. + + +type: scaled_float + +format: percent + +-- + +*`task_stats.cpu.system.ticks`*:: ++ +-- +CPU system ticks. + + +type: long + +-- + +*`task_stats.cpu.user.pct`*:: ++ +-- +Percentage of time in user space. + + +type: scaled_float + +format: percent + +-- + +*`task_stats.cpu.user.norm.pct`*:: ++ +-- +Percentage of time in user space normalized by the number of CPU cores. + + +type: scaled_float + +format: percent + +-- + +*`task_stats.cpu.user.ticks`*:: ++ +-- +CPU ticks in user space. + + +type: long + +-- + +*`task_stats.cpu.total.pct`*:: ++ +-- +Total CPU usage. + + +type: scaled_float + +format: percent + +-- + +*`task_stats.cpu.total.norm.pct`*:: ++ +-- +Total CPU usage normalized by the number of CPU cores. + + +type: scaled_float + +format: percent + +-- + +[float] +=== diskio + +Disk I/O metrics. + + +[float] +=== read + +Accumulated reads during the life of the container + + + +*`task_stats.diskio.read.ops`*:: ++ +-- +Number of reads during the life of the container + + +type: long + +-- + +*`task_stats.diskio.read.bytes`*:: ++ +-- +Bytes read during the life of the container + + +type: long + +format: bytes + +-- + +*`task_stats.diskio.read.rate`*:: ++ +-- +Number of current reads per second + + +type: long + +-- + +*`task_stats.diskio.read.service_time`*:: ++ +-- +Total time to service IO requests, in nanoseconds + + +type: long + +-- + +*`task_stats.diskio.read.wait_time`*:: ++ +-- +Total time requests spent waiting in queues for service, in nanoseconds + + +type: long + +-- + +*`task_stats.diskio.read.queued`*:: ++ +-- +Total number of queued requests + + +type: long + +-- + +*`task_stats.diskio.read.reads`*:: ++ +-- + +deprecated:[6.4] + +Number of current reads per second + + +type: scaled_float + +-- + +[float] +=== write + +Accumulated writes during the life of the container + + + +*`task_stats.diskio.read.write.ops`*:: ++ +-- +Number of writes during the life of the container + + +type: long + +-- + +*`task_stats.diskio.read.write.bytes`*:: ++ +-- +Bytes written during the life of the container + + +type: long + +format: bytes + +-- + +*`task_stats.diskio.read.write.rate`*:: ++ +-- +Number of current writes per second + + +type: long + +-- + +*`task_stats.diskio.read.write.service_time`*:: ++ +-- +Total time to service IO requests, in nanoseconds + + +type: long + +-- + +*`task_stats.diskio.read.write.wait_time`*:: ++ +-- +Total time requests spent waiting in queues for service, in nanoseconds + + +type: long + +-- + +*`task_stats.diskio.read.write.queued`*:: ++ +-- +Total number of queued requests + + +type: long + +-- + +*`task_stats.diskio.read.writes`*:: ++ +-- + +deprecated:[6.4] + +Number of current writes per second + + +type: scaled_float + +-- + +[float] +=== summary + +Accumulated reads and writes during the life of the container + + + +*`task_stats.diskio.read.summary.ops`*:: ++ +-- +Number of I/O operations during the life of the container + + +type: long + +-- + +*`task_stats.diskio.read.summary.bytes`*:: ++ +-- +Bytes read and written during the life of the container + + +type: long + +format: bytes + +-- + +*`task_stats.diskio.read.summary.rate`*:: ++ +-- +Number of current operations per second + + +type: long + +-- + +*`task_stats.diskio.read.summary.service_time`*:: ++ +-- +Total time to service IO requests, in nanoseconds + + +type: long + +-- + +*`task_stats.diskio.read.summary.wait_time`*:: ++ +-- +Total time requests spent waiting in queues for service, in nanoseconds + + +type: long + +-- + +*`task_stats.diskio.read.summary.queued`*:: ++ +-- +Total number of queued requests + + +type: long + +-- + +*`task_stats.diskio.read.total`*:: ++ +-- + +deprecated:[6.4] + +Number of reads and writes per second + + +type: scaled_float + +-- + +[float] +=== memory + +Memory metrics. + + +*`task_stats.memory.stats.*`*:: ++ +-- +Raw memory stats from the cgroups memory.stat interface + + +type: object + +-- + +[float] +=== network + +Network metrics. + + +*`task_stats.network.interface`*:: ++ +-- +Network interface name. + + +type: keyword + +-- + +[float] +=== inbound + +Incoming network stats since the container started. + + + +*`task_stats.network.inbound.bytes`*:: ++ +-- +Total number of incoming bytes. + + +type: long + +format: bytes + +-- + +*`task_stats.network.inbound.dropped`*:: ++ +-- +Total number of dropped incoming packets. + + +type: long + +-- + +*`task_stats.network.inbound.errors`*:: ++ +-- +Total errors on incoming packets. + + +type: long + +-- + +*`task_stats.network.inbound.packets`*:: ++ +-- +Total number of incoming packets. + + +type: long + +-- + +[float] +=== outbound + +Outgoing network stats since the container started. + + + +*`task_stats.network.outbound.bytes`*:: ++ +-- +Total number of outgoing bytes. + + +type: long + +format: bytes + +-- + +*`task_stats.network.outbound.dropped`*:: ++ +-- +Total number of dropped outgoing packets. + + +type: long + +-- + +*`task_stats.network.outbound.errors`*:: ++ +-- +Total errors on outgoing packets. + + +type: long + +-- + +*`task_stats.network.outbound.packets`*:: ++ +-- +Total number of outgoing packets. + + +type: long + +-- + [[exported-fields-azure]] == Azure fields diff --git a/metricbeat/docs/modules/awsfargate.asciidoc b/metricbeat/docs/modules/awsfargate.asciidoc new file mode 100644 index 00000000000..1de246049d7 --- /dev/null +++ b/metricbeat/docs/modules/awsfargate.asciidoc @@ -0,0 +1,86 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-awsfargate]] +[role="xpack"] +== AWS Fargate module + +beta[] + +Amazon ECS on Fargate provides a method to retrieve various metadata, network +metrics, and Docker stats about tasks and containers. This is referred to as the +https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html[task metadata endpoint] +and this endpoint is available per container. + +The environment variable is injected by default into the containers of Amazon +ECS tasks on Fargate that use platform version 1.4.0 or later and Amazon ECS +tasks on Amazon EC2 that are running at least version 1.39.0 of the Amazon ECS +container agent. + +The awsfargate module is a Metricbeat module which collects AWS fargate metrics +from task metadata endpoint. + +[float] +== Introduction to AWS ECS and Fargate +Amazon Elastic Container Service (Amazon ECS) is a highly scalable, fast, +container management service that makes it easy to run, stop, and manage +containers. ECS has two launch types that can define how compute resources will +be managed: ECS EC2 and ECS Fargate. + +* *ECS EC2* + +ECS EC2 launches containers that run on EC2 instances. Users have to manage EC2 +instances. Pricing depends on the number of EC2 instances running. + +* *ECS Fargate* + +ECS Fargate removes the responsibility of provisioning, configuring, and +managing the EC2 instances by allowing AWS to manage the EC2 instances. Users +only need to specify containers and tasks. Pricing based on the number of tasks. + +[float] +== Task Metadata Endpoint +https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html[Task metadata endpoint] +returns https://docs.docker.com/engine/api/v1.30/#operation/ContainerStats[Docker stats] +in JSON format for all the containers associated with the task. +This endpoint is only available from within the task definition itself, which +means Metricbeat needs to be run as a sidecar container within the task +definition. Since the metadata endpoint is only accessible from within the +Fargate Task, there is no authentication in place. + +[float] +== Metricsets +Currently, we have `task_stats` metricset in `awsfargate` module. + +[float] +=== `task_stats` +This metricset collects runtime CPU metrics, disk I/O metrics, memory metrics, +network metrics and container metadata from both endpoint +`${ECS_CONTAINER_METADATA_URI_V4}/task/stats` and `${ECS_CONTAINER_METADATA_URI_V4}/task`. + + +[float] +=== Example configuration + +The AWS Fargate module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: awsfargate + period: 10s + metricsets: + - task_stats +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::awsfargate/task_stats.asciidoc[] + diff --git a/metricbeat/docs/modules/awsfargate/task_stats.asciidoc b/metricbeat/docs/modules/awsfargate/task_stats.asciidoc new file mode 100644 index 00000000000..9e43c8a2cea --- /dev/null +++ b/metricbeat/docs/modules/awsfargate/task_stats.asciidoc @@ -0,0 +1,25 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-awsfargate-task_stats]] +[role="xpack"] +=== AWS Fargate task_stats metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 68476e1acaa..847a13703ce 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -32,6 +32,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> beta[] +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .11+| .11+| |<> beta[] |<> beta[] @@ -298,6 +300,7 @@ include::modules/aerospike.asciidoc[] include::modules/apache.asciidoc[] include::modules/appsearch.asciidoc[] include::modules/aws.asciidoc[] +include::modules/awsfargate.asciidoc[] include::modules/azure.asciidoc[] include::modules/beat.asciidoc[] include::modules/ceph.asciidoc[] diff --git a/metricbeat/module/docker/cpu/cpu_test.go b/metricbeat/module/docker/cpu/cpu_test.go index d14fda678d9..d599abd19ed 100644 --- a/metricbeat/module/docker/cpu/cpu_test.go +++ b/metricbeat/module/docker/cpu/cpu_test.go @@ -29,8 +29,8 @@ import ( var cpuService CPUService -func cpuUsageFor(stats types.StatsJSON) *cpuUsage { - u := cpuUsage{ +func cpuUsageFor(stats types.StatsJSON) *CPUUsage { + u := CPUUsage{ Stat: &docker.Stat{Stats: stats}, systemDelta: 1000000000, // Nanoseconds in a second } diff --git a/metricbeat/module/docker/cpu/helper.go b/metricbeat/module/docker/cpu/helper.go index 75527285f1e..fcb8dc2de55 100644 --- a/metricbeat/module/docker/cpu/helper.go +++ b/metricbeat/module/docker/cpu/helper.go @@ -62,7 +62,7 @@ func (c *CPUService) getCPUStatsList(rawStats []docker.Stat, dedot bool) []CPUSt } func (c *CPUService) getCPUStats(myRawStat *docker.Stat, dedot bool) CPUStats { - usage := cpuUsage{Stat: myRawStat} + usage := CPUUsage{Stat: myRawStat} stats := CPUStats{ Time: common.Time(myRawStat.Stats.Read), @@ -89,7 +89,7 @@ func (c *CPUService) getCPUStats(myRawStat *docker.Stat, dedot bool) CPUStats { // TODO: These helper should be merged with the cpu helper in system/cpu -type cpuUsage struct { +type CPUUsage struct { *docker.Stat cpus uint32 @@ -98,7 +98,7 @@ type cpuUsage struct { // CPUS returns the number of cpus. If number of cpus is equal to zero, the field will // be updated/initialized with the corresponding value retrieved from Docker API. -func (u *cpuUsage) CPUs() uint32 { +func (u *CPUUsage) CPUs() uint32 { if u.cpus == 0 { if u.Stats.CPUStats.OnlineCPUs != 0 { u.cpus = u.Stats.CPUStats.OnlineCPUs @@ -119,7 +119,7 @@ func (u *cpuUsage) CPUs() uint32 { } // SystemDelta calculates system delta. -func (u *cpuUsage) SystemDelta() uint64 { +func (u *CPUUsage) SystemDelta() uint64 { if u.systemDelta == 0 { u.systemDelta = u.Stats.CPUStats.SystemUsage - u.Stats.PreCPUStats.SystemUsage } @@ -127,7 +127,7 @@ func (u *cpuUsage) SystemDelta() uint64 { } // PerCPU calculates per CPU usage. -func (u *cpuUsage) PerCPU() common.MapStr { +func (u *CPUUsage) PerCPU() common.MapStr { var output common.MapStr if len(u.Stats.CPUStats.CPUUsage.PercpuUsage) == len(u.Stats.PreCPUStats.CPUUsage.PercpuUsage) { output = common.MapStr{} @@ -151,42 +151,42 @@ func (u *cpuUsage) PerCPU() common.MapStr { } // TotalNormalized calculates total CPU usage normalized. -func (u *cpuUsage) Total() float64 { +func (u *CPUUsage) Total() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.TotalUsage, u.Stats.PreCPUStats.CPUUsage.TotalUsage, u.CPUs()) } // TotalNormalized calculates total CPU usage normalized by the number of CPU cores. -func (u *cpuUsage) TotalNormalized() float64 { +func (u *CPUUsage) TotalNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.TotalUsage, u.Stats.PreCPUStats.CPUUsage.TotalUsage, 1) } // InKernelMode calculates percentage of time in kernel space. -func (u *cpuUsage) InKernelMode() float64 { +func (u *CPUUsage) InKernelMode() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInKernelmode, u.Stats.PreCPUStats.CPUUsage.UsageInKernelmode, u.CPUs()) } // InKernelModeNormalized calculates percentage of time in kernel space normalized by the number of CPU cores. -func (u *cpuUsage) InKernelModeNormalized() float64 { +func (u *CPUUsage) InKernelModeNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInKernelmode, u.Stats.PreCPUStats.CPUUsage.UsageInKernelmode, 1) } // InUserMode calculates percentage of time in user space. -func (u *cpuUsage) InUserMode() float64 { +func (u *CPUUsage) InUserMode() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInUsermode, u.Stats.PreCPUStats.CPUUsage.UsageInUsermode, u.CPUs()) } // InUserModeNormalized calculates percentage of time in user space normalized by the number of CPU cores. -func (u *cpuUsage) InUserModeNormalized() float64 { +func (u *CPUUsage) InUserModeNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.CPUUsage.UsageInUsermode, u.Stats.PreCPUStats.CPUUsage.UsageInUsermode, 1) } // System calculates percentage of total CPU time in the system. -func (u *cpuUsage) System() float64 { +func (u *CPUUsage) System() float64 { return u.calculatePercentage(u.Stats.CPUStats.SystemUsage, u.Stats.PreCPUStats.SystemUsage, u.CPUs()) } // SystemNormalized calculates percentage of total CPU time in the system, normalized by the number of CPU cores. -func (u *cpuUsage) SystemNormalized() float64 { +func (u *CPUUsage) SystemNormalized() float64 { return u.calculatePercentage(u.Stats.CPUStats.SystemUsage, u.Stats.PreCPUStats.SystemUsage, 1) } @@ -194,7 +194,7 @@ func (u *cpuUsage) SystemNormalized() float64 { // The "oldValue" refers to the CPU statistics of the last read. // Time here is expressed by second and not by nanoseconde. // The main goal is to expose the %, in the same way, it's displayed by docker Client. -func (u *cpuUsage) calculatePercentage(newValue uint64, oldValue uint64, numCPUS uint32) float64 { +func (u *CPUUsage) calculatePercentage(newValue uint64, oldValue uint64, numCPUS uint32) float64 { if newValue < oldValue { logp.Err("Error calculating CPU time change for docker module: new stats value (%v) is lower than the old one(%v)", newValue, oldValue) return -1 diff --git a/metricbeat/scripts/mage/docs_collector.go b/metricbeat/scripts/mage/docs_collector.go index b58bad3edae..b505db79a69 100644 --- a/metricbeat/scripts/mage/docs_collector.go +++ b/metricbeat/scripts/mage/docs_collector.go @@ -158,7 +158,7 @@ func getDefaultMetricsets() (map[string][]string, error) { return nil, err } for k, v := range msetMap { - masterMap[k] = v + masterMap[k] = append(masterMap[k], v...) } } diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index f4a2c1c6cfc..15a643ec929 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -17,6 +17,8 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/ec2" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/rds" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/sqs" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/awsfargate" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/awsfargate/task_stats" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/app_insights" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/billing" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 164dc564f2f..0a6a954ec6f 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -237,6 +237,12 @@ metricbeat.modules: - usage - vpn +#----------------------------- AWS Fargate Module ----------------------------- +- module: awsfargate + period: 10s + metricsets: + - task_stats + #-------------------------------- Azure Module -------------------------------- - module: azure metricsets: diff --git a/x-pack/metricbeat/module/awsfargate/_meta/config.yml b/x-pack/metricbeat/module/awsfargate/_meta/config.yml new file mode 100644 index 00000000000..b83eee4cccf --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/_meta/config.yml @@ -0,0 +1,4 @@ +- module: awsfargate + period: 10s + metricsets: + - task_stats diff --git a/x-pack/metricbeat/module/awsfargate/_meta/docs.asciidoc b/x-pack/metricbeat/module/awsfargate/_meta/docs.asciidoc new file mode 100644 index 00000000000..e0f7cb88e23 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/_meta/docs.asciidoc @@ -0,0 +1,50 @@ +Amazon ECS on Fargate provides a method to retrieve various metadata, network +metrics, and Docker stats about tasks and containers. This is referred to as the +https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html[task metadata endpoint] +and this endpoint is available per container. + +The environment variable is injected by default into the containers of Amazon +ECS tasks on Fargate that use platform version 1.4.0 or later and Amazon ECS +tasks on Amazon EC2 that are running at least version 1.39.0 of the Amazon ECS +container agent. + +The awsfargate module is a Metricbeat module which collects AWS fargate metrics +from task metadata endpoint. + +[float] +== Introduction to AWS ECS and Fargate +Amazon Elastic Container Service (Amazon ECS) is a highly scalable, fast, +container management service that makes it easy to run, stop, and manage +containers. ECS has two launch types that can define how compute resources will +be managed: ECS EC2 and ECS Fargate. + +* *ECS EC2* + +ECS EC2 launches containers that run on EC2 instances. Users have to manage EC2 +instances. Pricing depends on the number of EC2 instances running. + +* *ECS Fargate* + +ECS Fargate removes the responsibility of provisioning, configuring, and +managing the EC2 instances by allowing AWS to manage the EC2 instances. Users +only need to specify containers and tasks. Pricing based on the number of tasks. + +[float] +== Task Metadata Endpoint +https://docs.aws.amazon.com/AmazonECS/latest/userguide/task-metadata-endpoint-v4-fargate.html[Task metadata endpoint] +returns https://docs.docker.com/engine/api/v1.30/#operation/ContainerStats[Docker stats] +in JSON format for all the containers associated with the task. +This endpoint is only available from within the task definition itself, which +means Metricbeat needs to be run as a sidecar container within the task +definition. Since the metadata endpoint is only accessible from within the +Fargate Task, there is no authentication in place. + +[float] +== Metricsets +Currently, we have `task_stats` metricset in `awsfargate` module. + +[float] +=== `task_stats` +This metricset collects runtime CPU metrics, disk I/O metrics, memory metrics, +network metrics and container metadata from both endpoint +`${ECS_CONTAINER_METADATA_URI_V4}/task/stats` and `${ECS_CONTAINER_METADATA_URI_V4}/task`. diff --git a/x-pack/metricbeat/module/awsfargate/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/_meta/fields.yml new file mode 100644 index 00000000000..11242ff2de3 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/_meta/fields.yml @@ -0,0 +1,6 @@ +- key: awsfargate + title: "AWS Fargate" + description: > + `awsfargate` module collects AWS fargate metrics from task metadata endpoint. + release: beta + fields: diff --git a/x-pack/metricbeat/module/awsfargate/awsfargate.go b/x-pack/metricbeat/module/awsfargate/awsfargate.go new file mode 100644 index 00000000000..94a030b90ca --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/awsfargate.go @@ -0,0 +1,54 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awsfargate + +import ( + "time" + + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// Config defines all required and optional parameters for awsfargate metricsets +type Config struct { + Period time.Duration `config:"period" validate:"nonzero,required"` +} + +// MetricSet is the base metricset for all aws metricsets +type MetricSet struct { + mb.BaseMetricSet + Period time.Duration +} + +// ModuleName is the name of this module. +const ModuleName = "awsfargate" + +func init() { + if err := mb.Registry.AddModule(ModuleName, newModule); err != nil { + panic(err) + } +} + +func newModule(base mb.BaseModule) (mb.Module, error) { + var config Config + if err := base.UnpackConfig(&config); err != nil { + return nil, err + } + return &base, nil +} + +// NewMetricSet creates a base metricset for awsfargate metricsets +func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + var config Config + err := base.Module().UnpackConfig(&config) + if err != nil { + return nil, err + } + + metricSet := MetricSet{ + BaseMetricSet: base, + Period: config.Period, + } + return &metricSet, nil +} diff --git a/x-pack/metricbeat/module/awsfargate/cloudformation.yml b/x-pack/metricbeat/module/awsfargate/cloudformation.yml new file mode 100644 index 00000000000..18e0759414a --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/cloudformation.yml @@ -0,0 +1,82 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + SubnetID: + Type: String +Resources: + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: metricbeat-cloudformation-fargate + ClusterSettings: + - Name: containerInsights + Value: enabled + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: metricbeat-fargate-log-group + ExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: ecsFargateTaskExecutionRole + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub 'EcsTaskExecutionRole-${AWS::StackName}' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - + - + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: deployment-task-metricbeat + Cpu: 256 + Memory: 512 + NetworkMode: awsvpc + ExecutionRoleArn: !Ref ExecutionRole + ContainerDefinitions: + - Name: deployment-task-metricbeat-container + Image: kaiyansheng/metricbeat-awsfargate:v1 + Secrets: + - Name: ELASTIC_CLOUD_ID + ValueFrom: + - Name: ELASTIC_CLOUD_AUTH + ValueFrom: + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: ecs + EntryPoint: + - sh + - -c + Command: + - ./metricbeat setup && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + RequiresCompatibilities: + - EC2 + - FARGATE + Service: + Type: AWS::ECS::Service + Properties: + ServiceName: deployment-metricbeat-service + Cluster: !Ref Cluster + TaskDefinition: !Ref TaskDefinition + DesiredCount: 1 + LaunchType: FARGATE + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + Subnets: + - !Ref SubnetID diff --git a/x-pack/metricbeat/module/awsfargate/doc.go b/x-pack/metricbeat/module/awsfargate/doc.go new file mode 100644 index 00000000000..516d3548dd2 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/doc.go @@ -0,0 +1,5 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awsfargate diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go new file mode 100644 index 00000000000..9659165097f --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package awsfargate + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "awsfargate", asset.ModuleFieldsPri, AssetAwsfargate); err != nil { + panic(err) + } +} + +// AssetAwsfargate returns asset data. +// This is the base64 encoded gzipped contents of module/awsfargate. +func AssetAwsfargate() string { + return "eJzsWk1z2zYQvftX7Hh6yjTKJdODDp1RnWTGB9sZfzRHGQJXMioSYAAwitLpf+8ABCRKAimQsmTWDY8iwH1v9+0uPvQW5rgcAlmoKZEzovEMQDOd4hDOR1/u4FP56/kZQIKKSpZrJvgQfj8DAHhcz3uETCRFikBFmiLVCsx09xIy1JJRBVMpMtBEzc0vJCGaAPIkF4zrwRmAxBSJwiFMUJMzgCnDNFFDa8s8b4GTDIf2C2OliVarVwB6meMQZlIUeeXXAGr/PK4/8whUcE0YV6Cf1nD1E9GwQImgqCQ5JiWBKjNLxn4Dfvn748Xd+OLm+n50ef3xdnz18X70YXQ/Gj/cXo7/fP/POzP2XTk2SN8/227wz7Y7qi6hebHxe50/dnxyW3DNMoSLzw+e92BrfMhu1fYcJcd0kFO9M8TDUJSkmIynqSChQVMhM6KHkKOkyEMjGuLon8/lZDJDEFOwpBh34EDlhOI2sR0KXMjsP8ADDE6Ssh+YwGRpJcuLbILSTDCBpELiThh32GpG56qWair4rBsBg8B+O97/aqk09s71QpPUsSmDYDztoO6j0kcpNfA5VFKO9TEl5YBaE/VACoWyb3533jbQ9iWChd9H7exwOFQxlulJSlCM321uvLjP71cZWigy2wu3FzrZwtxFFp5VwtSciW7rmA9MzeHy3U3XRYxEktT6MWQ90j0jSousSInGxNpQkBSS8Zn1TMqmZXY9oV+BogwFqgZ8lYDIQ4kE+5IpkoZ5rldx7ExkDXey1NgZsBdu00ciSf1hPmEJHcZHlpunTnRa+58WUiLXLg65KW9IBQ8JuNKiUX5jFMemlB8ZaVkTbM/QwhuGyxuQ+LVApdWvpi5zwkWJOxxDD3xBmD4xao8TVG78bBAYaTAOXwssUBkFel6tuNjp4Tg9N5F12S2Nrkg169hIqhHfnqZS4swlUlPzhvDb4P2Lyn4h2Z7MrKvuLTBWq7w12LE6Npd6iCr3EKOkFtxgIwaHsYur/9EMYvpAS6ZlPzAsNfLDaTa0hWiWneLkc8XFa0+yQLs+cQzoz9MzoEXfODKLZ+8hENtHjkesW08pNdjzphKXKKskKbKMyOUp+0rZ9Qh/7R3GbOJEjpKYaa++09idhw/q62g5leD9bDsnZ/Gz7WwckvW06+wU85pM8VwyzMROu4k8Gruyc7sejNkLy8Gb2rMxMfkLg0eQ5YtxoyIqY8YZyXPGZ27C+Zvzbgdut2ThvOXuZctLZ1NCraeUezswb4FxjXJKaChZY07dqMgyVqedZ1sGXFgr2h6pGrEIDl8YT8SiLmNje3xTisAL9sYy7Zs/6DnkSOY9pPAZydzJIzpwK0qSfSMaxwsh5yYlFOrB/mp2mgNThw0cNlCoo3lNCUsHVBTBuwZoV5ajwH4iLAVrEOXubUoVWsr25fFp3OuKtYXTjFiq5i3VM9Sd27u7jVoaBvRayo1jKlGxxCydFGpQ7EfgGm6bV/gObpNVhKgh6k6uA7+rGmbeSvCucZukvd7bS7NJdS1RP9j7xBj9QYQGq1Qy8r1xXKQSoYUaoR1381yR7559zXVw9YnTIrTVI7TSJHRgWWEYpUdoUVCgJ6HcKDCBYHo+HLXpq93W+dfl5K4L/aalcIlgjsuFkKHtX4Q7PLqVGWu2/i8OjE9EETw3OPhO/pJTkZm1i3O32ygoxiluHraYN1JjElJjzPagL7fc29to5j1gv928zkikyPOT31k6q2ukOaFzrGsBHitKKeSx/wdRQi1NmZVvO4hu0InduR/j6ti30EdLvJtCz8T/OvGE90DvE2+FtL+J1w7iyyTeLsZ/AwAA//8SjLaw" +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json new file mode 100644 index 00000000000..98a48adecbb --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json @@ -0,0 +1,148 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "awsfargate": { + "task_stats": { + "cpu": { + "core": null, + "kernel": { + "norm": { + "pct": 0 + }, + "pct": 0, + "ticks": 1520000000 + }, + "system": { + "norm": { + "pct": 1 + }, + "pct": 2, + "ticks": 1420180000000 + }, + "total": { + "norm": { + "pct": 0.2 + }, + "pct": 0.4 + }, + "user": { + "norm": { + "pct": 0 + }, + "pct": 0, + "ticks": 490000000 + } + }, + "diskio": { + "read": { + "bytes": 3452928, + "ops": 118, + "queued": 0, + "rate": 0, + "service_time": 0, + "wait_time": 0 + }, + "reads": 0, + "summary": { + "bytes": 3452928, + "ops": 118, + "queued": 0, + "rate": 0, + "service_time": 0, + "wait_time": 0 + }, + "total": 0, + "write": { + "bytes": 0, + "ops": 0, + "queued": 0, + "rate": 0, + "service_time": 0, + "wait_time": 0 + }, + "writes": 0 + }, + "memory": { + "fail": { + "count": 0 + }, + "limit": 0, + "rss": { + "pct": 0.0010557805807105247, + "total": 4157440 + }, + "stats": { + "active_anon": 4157440, + "active_file": 4497408, + "cache": 6000640, + "dirty": 16384, + "hierarchical_memory_limit": 2147483648, + "hierarchical_memsw_limit": 9223372036854772000, + "inactive_anon": 0, + "inactive_file": 1503232, + "mapped_file": 2183168, + "pgfault": 6668, + "pgmajfault": 52, + "pgpgin": 5925, + "pgpgout": 3445, + "rss": 4157440, + "rss_huge": 0, + "total_active_anon": 4157440, + "total_active_file": 4497408, + "total_cache": 600064, + "total_dirty": 16384, + "total_inactive_anon": 0, + "total_inactive_file": 4497408, + "total_mapped_file": 2183168, + "total_pgfault": 6668, + "total_pgmajfault": 52, + "total_pgpgin": 5925, + "total_pgpgout": 3445, + "total_rss": 4157440, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "usage": { + "max": 15294464, + "pct": 0.003136136404770672, + "total": 12349440 + } + }, + "network": { + "eth0": { + "inbound": { + "bytes": 137315578, + "dropped": 0, + "errors": 0, + "packets": 94338 + }, + "outbound": { + "bytes": 1086811, + "dropped": 0, + "errors": 0, + "packets": 25857 + } + } + } + } + }, + "container": { + "id": "query-metadata-1", + "image": { + "name": "mreferre/eksutils" + }, + "labels": { + "com_amazonaws_ecs_cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com_amazonaws_ecs_container-name": "query-metadata", + "com_amazonaws_ecs_task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", + "com_amazonaws_ecs_task-definition-family": "query-metadata", + "com_amazonaws_ecs_task-definition-version": "7" + }, + "name": "query-metadata" + }, + "service": { + "type": "awsfargate" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc new file mode 100644 index 00000000000..6d5983fe63d --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/docs.asciidoc @@ -0,0 +1,153 @@ +The `task_stats` metricset in `awsfargate` module allows users to monitor +containers inside the same AWS Fargate task. It fetches runtime CPU metrics, +disk I/O metrics, memory metrics, network metrics and container metadata from +both endpoint `${ECS_CONTAINER_METADATA_URI_V4}/task/stats` and +`${ECS_CONTAINER_METADATA_URI_V4}/task`. + +[float] +=== Configuration Example +This metricset should be ran as a sidecar inside the same AWS Fargate task +definition, and the default configuration file should work. + +[source,yaml] +---- +- module: awsfargate + period: 10s + metricsets: + - task_stats +---- + +[float] +=== Setup Metricbeat Using AWS Fargate +This section is to provide users an AWS native way of configuring Fargate task +definition to run application containers and Metricbeat container using AWS +CloudFormation. + +[float] +==== Store Elastic Cloud Credentials into AWS Secret Manager +If users are using Elastic Cloud, it's recommended to store cloud id and cloud +auth into AWS secret manager. Here are the AWS CLI example: + +Create secret ELASTIC_CLOUD_AUTH: +---- +aws --region us-east-1 secretsmanager create-secret --name ELASTIC_CLOUD_AUTH --secret-string XXX +---- + +Create secret ELASTIC_CLOUD_ID: +---- +aws --region us-east-1 secretsmanager create-secret --name ELASTIC_CLOUD_ID --secret-string YYYY +---- + +[float] +==== AWS CloudFormation Template Example +Here is an example of AWS CloudFormation template to create a new cluster, +create a task definition that runs Metricbeat container and start the service. +Please copy this section into a `cloud_formation.yml` file locally and replace +``, ``, and `` with +your own preferred names. Also you can find `` and +`` values from AWS secret manager. + +[source,yaml] +---- +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + SubnetID: + Type: String +Resources: + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: + ClusterSettings: + - Name: containerInsights + Value: enabled + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: + ExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: ecsFargateTaskExecutionRole + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub 'EcsTaskExecutionRole-${AWS::StackName}' + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - + - + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: deployment-task-metricbeat + Cpu: 256 + Memory: 512 + NetworkMode: awsvpc + ExecutionRoleArn: !Ref ExecutionRole + ContainerDefinitions: + - Name: deployment-task-metricbeat-container + Image: kaiyansheng/metricbeat-awsfargate:v1 + Secrets: + - Name: ELASTIC_CLOUD_ID + ValueFrom: + - Name: ELASTIC_CLOUD_AUTH + ValueFrom: + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: ecs + EntryPoint: + - sh + - -c + Command: + - ./metricbeat setup && ./metricbeat modules disable system && ./metricbeat modules enable awsfargate && ./metricbeat -e -E cloud.id=$ELASTIC_CLOUD_ID -E cloud.auth=$ELASTIC_CLOUD_AUTH + RequiresCompatibilities: + - EC2 + - FARGATE + Service: + Type: AWS::ECS::Service + Properties: + ServiceName: + Cluster: !Ref Cluster + TaskDefinition: !Ref TaskDefinition + DesiredCount: 1 + LaunchType: FARGATE + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + Subnets: + - !Ref SubnetID +---- + +[float] +==== Create CloudFormation Stack +Here is the AWS CLI to create a stack using the CloudFormation config file above: +---- +aws --region us-east-1 cloudformation create-stack --stack-name --template-body file://./cloudformation.yml --capabilities CAPABILITY_NAMED_IAM --parameters 'ParameterKey=SubnetID,ParameterValue=' +---- + +Make sure to replace `` with your own subnet in this command. Please go +to Services -> VPC -> Subnets to find subnet ID to use. You can also add several +more containers under the TaskDefinition section. + +[float] +==== Delete CloudFormation Stack +Here is the AWS CLI to delete a stack including the cluster, task definition and +all containers: +---- +aws cloudformation delete-stack --stack-name +---- diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml new file mode 100644 index 00000000000..e0099191f35 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -0,0 +1,298 @@ +- name: task_stats + type: group + description: > + `task_stats` contains the metrics that were scraped from AWS fargate task stats ${ECS_CONTAINER_METADATA_URI_V4}/task/stats metadata endpoint. + release: beta + fields: + - name: cpu + type: group + description: Runtime CPU metrics. + fields: + - name: kernel.pct + type: scaled_float + format: percent + description: > + Percentage of time in kernel space. + - name: kernel.norm.pct + type: scaled_float + format: percent + description: > + Percentage of time in kernel space normalized by the number of CPU cores. + - name: kernel.ticks + type: long + description: > + CPU ticks in kernel space. + - name: system.pct + type: scaled_float + format: percent + description: > + Percentage of total CPU time in the system. + - name: system.norm.pct + type: scaled_float + format: percent + description: > + Percentage of total CPU time in the system normalized by the number of CPU cores. + - name: system.ticks + type: long + description: > + CPU system ticks. + - name: user.pct + type: scaled_float + format: percent + description: > + Percentage of time in user space. + - name: user.norm.pct + type: scaled_float + format: percent + description: > + Percentage of time in user space normalized by the number of CPU cores. + - name: user.ticks + type: long + description: > + CPU ticks in user space. + - name: total.pct + type: scaled_float + format: percent + description: > + Total CPU usage. + - name: total.norm.pct + type: scaled_float + format: percent + description: > + Total CPU usage normalized by the number of CPU cores. + - name: diskio + type: group + description: Disk I/O metrics. + fields: + - name: read + type: group + description: > + Accumulated reads during the life of the container + fields: + - name: ops + type: long + description: > + Number of reads during the life of the container + - name: bytes + type: long + format: bytes + description: > + Bytes read during the life of the container + - name: rate + type: long + description: > + Number of current reads per second + - name: service_time + type: long + description: > + Total time to service IO requests, in nanoseconds + - name: wait_time + type: long + description: > + Total time requests spent waiting in queues for service, in nanoseconds + - name: queued + type: long + description: > + Total number of queued requests + - name: reads + type: scaled_float + deprecated: 6.4 + description: > + Number of current reads per second + - name: write + type: group + description: > + Accumulated writes during the life of the container + fields: + - name: ops + type: long + description: > + Number of writes during the life of the container + - name: bytes + type: long + format: bytes + description: > + Bytes written during the life of the container + - name: rate + type: long + description: > + Number of current writes per second + - name: service_time + type: long + description: > + Total time to service IO requests, in nanoseconds + - name: wait_time + type: long + description: > + Total time requests spent waiting in queues for service, in nanoseconds + - name: queued + type: long + description: > + Total number of queued requests + - name: writes + type: scaled_float + deprecated: 6.4 + description: > + Number of current writes per second + - name: summary + type: group + description: > + Accumulated reads and writes during the life of the container + fields: + - name: ops + type: long + description: > + Number of I/O operations during the life of the container + - name: bytes + type: long + format: bytes + description: > + Bytes read and written during the life of the container + - name: rate + type: long + description: > + Number of current operations per second + - name: service_time + type: long + description: > + Total time to service IO requests, in nanoseconds + - name: wait_time + type: long + description: > + Total time requests spent waiting in queues for service, in nanoseconds + - name: queued + type: long + description: > + Total number of queued requests + - name: total + type: scaled_float + deprecated: 6.4 + description: > + Number of reads and writes per second + - name: memory + type: group + description: Memory metrics. + fields: + - name: stats.* + type: object + object_type: long + object_type_mapping_type: "*" + description: > + Raw memory stats from the cgroups memory.stat interface + fields: + - name: commit + type: group + description: > + Committed bytes on Windows + fields: + - name: total + type: long + format: bytes + description: > + Total bytes + - name: peak + type: long + format: bytes + description: > + Peak committed bytes on Windows + - name: private_working_set.total + type: long + format: bytes + description: > + private working sets on Windows + - name: fail.count + type: scaled_float + description: > + Fail counter. + - name: limit + type: long + format: bytes + description: > + Memory limit. + - name: rss + type: group + description: > + RSS memory stats. + fields: + - name: total + type: long + format: bytes + description: > + Total memory resident set size. + - name: pct + type: scaled_float + format: percent + description: > + Memory resident set size percentage. + - name: usage + type: group + description: > + Usage memory stats. + fields: + - name: max + type: long + format: bytes + description: > + Max memory usage. + - name: pct + type: scaled_float + format: percent + description: > + Memory usage percentage. + - name: total + type: long + format: bytes + description: > + Total memory usage. + - name: network + type: group + description: Network metrics. + fields: + - name: interface + type: keyword + description: > + Network interface name. + - name: inbound + type: group + description: > + Incoming network stats since the container started. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of incoming bytes. + - name: dropped + type: long + description: > + Total number of dropped incoming packets. + - name: errors + type: long + description: > + Total errors on incoming packets. + - name: packets + type: long + description: > + Total number of incoming packets. + - name: outbound + type: group + description: > + Outgoing network stats since the container started. + fields: + - name: bytes + type: long + format: bytes + description: > + Total number of outgoing bytes. + - name: dropped + type: long + description: > + Total number of dropped outgoing packets. + - name: errors + type: long + description: > + Total errors on outgoing packets. + - name: packets + type: long + description: > + Total number of outgoing packets. diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json new file mode 100644 index 00000000000..65219722319 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json @@ -0,0 +1,17 @@ +{ + "Cluster": "arn:aws:ecs:us-west-2:123:cluster/default", + "TaskARN": "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", + "Family": "query-metadata-1", + "Revision": "7", + "Containers": [{ + "DockerId": "query-metadata-1", + "Name": "query-metadata", + "Image": "mreferre/eksutils", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "query-metadata", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", + "com.amazonaws.ecs.task-definition-family": "query-metadata", + "com.amazonaws.ecs.task-definition-version": "7"} + }] +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json new file mode 100644 index 00000000000..88ddceeb6b4 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task_stats.json @@ -0,0 +1,119 @@ +{ + "query-metadata-1": { + "blkio_stats": { + "io_service_bytes_recursive": [ + {"major": 202, "minor": 26368, "op": "Read", "value": 3452928}, + {"major": 202, "minor": 26368, "op": "Write", "value": 0}, + {"major": 202, "minor": 26368, "op": "Sync", "value": 3452928}, + {"major": 202, "minor": 26368, "op": "Async", "value": 0}, + {"major": 202, "minor": 26368, "op": "Total", "value": 3452928} + ], + "io_serviced_recursive": [ + {"major": 202, "minor": 26368, "op": "Read", "value": 118}, + {"major": 202, "minor": 26368, "op": "Write", "value": 0}, + {"major": 202, "minor": 26368, "op": "Sync", "value": 118}, + {"major": 202, "minor": 26368, "op": "Async", "value": 0}, + {"major": 202, "minor": 26368, "op": "Total", "value": 118} + ], + "io_queue_recursive": [], + "io_service_time_recursive": [], + "io_wait_time_recursive": [], + "io_merged_recursive": [], + "io_time_recursive": [], + "sectors_recursive": []}, + "cpu_stats": { + "cpu_usage": { + "percpu_usage": [1800000000, 500000000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "total_usage": 2300000000, + "usage_in_kernelmode": 1520000000, + "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1420180000000, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "id": "query-metadata-1", + "memory_stats": { + "limit": 3937787904, + "max_usage": 15294464, + "stats": { + "active_anon": 4157440, + "active_file": 4497408, + "cache": 6000640, + "dirty": 16384, + "hierarchical_memory_limit": 2147483648, + "hierarchical_memsw_limit": 9223372036854772000, + "inactive_anon": 0, + "inactive_file": 1503232, + "mapped_file": 2183168, + "pgfault": 6668, + "pgmajfault": 52, + "pgpgin": 5925, + "pgpgout": 3445, + "rss": 4157440, + "rss_huge": 0, + "total_active_anon": 4157440, + "total_active_file": 4497408, + "total_cache": 600064, + "total_dirty": 16384, + "total_inactive_anon": 0, + "total_inactive_file": 4497408, + "total_mapped_file": 2183168, + "total_pgfault": 6668, + "total_pgmajfault": 52, + "total_pgpgin": 5925, + "total_pgpgout": 3445, + "total_rss": 4157440, + "total_rss_huge": 0, + "total_unevictable": 0, + "total_writeback": 0, + "unevictable": 0, + "writeback": 0 + }, + "usage": 12349440 + }, + "name": "query-metadata-1", + "network_rate_stats": { + "rx_bytes_per_sec": 9.6001425, + "tx_bytes_per_sec": 8.7001295 + }, + "networks": { + "eth0": { + "rx_bytes": 137315578, + "rx_dropped": 0, + "rx_errors": 0, + "rx_packets": 94338, + "tx_bytes": 1086811, + "tx_dropped": 0, + "tx_errors": 0, + "tx_packets": 25857 + } + }, + "num_procs": 0, + "pids_stats": { + "current": 8 + }, + "precpu_stats": { + "cpu_usage": { + "percpu_usage": [1600000000, 300000000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "total_usage": 1900000000, + "usage_in_kernelmode": 1520000000, + "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1418180000000, + "throttling_data": { + "periods": 0, + "throttled_periods": 0, + "throttled_time": 0 + } + }, + "preread": "2020-10-28T01:00:08.03754086Z", + "read": "2020-10-28T01:00:09.044989605Z", + "storage_stats": {} + } +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go new file mode 100644 index 00000000000..0a21b9c0744 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "github.com/elastic/beats/v7/libbeat/common" + helpers "github.com/elastic/beats/v7/libbeat/common/docker" +) + +// container is a struct representation of a container +type container struct { + DockerId string + Name string + Image string + Labels map[string]string +} + +// ContainerMetadata is an struct represents container metadata +type ContainerMetadata struct { + Cluster string + TaskARN string + Family string + Revision string + Container *container +} + +func getContainerStats(c *container) *container { + return &container{ + DockerId: c.DockerId, + Image: c.Image, + Name: helpers.ExtractContainerName([]string{c.Name}), + Labels: deDotLabels(c.Labels), + } +} + +func deDotLabels(labels map[string]string) map[string]string { + outputLabels := map[string]string{} + for k, v := range labels { + // This is necessary so that ES does not interpret '.' fields as new + // nested JSON objects, and also makes this compatible with ES 2.x. + label := common.DeDot(k) + outputLabels[label] = v + } + + return outputLabels +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/cpu.go b/x-pack/metricbeat/module/awsfargate/task_stats/cpu.go new file mode 100644 index 00000000000..e10560c47cf --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/cpu.go @@ -0,0 +1,30 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "github.com/docker/docker/api/types" + + "github.com/elastic/beats/v7/metricbeat/module/docker" + "github.com/elastic/beats/v7/metricbeat/module/docker/cpu" +) + +func getCPUStats(taskStats types.StatsJSON) cpu.CPUStats { + usage := cpu.CPUUsage{Stat: &docker.Stat{Stats: taskStats}} + + return cpu.CPUStats{ + TotalUsage: usage.Total(), + TotalUsageNormalized: usage.TotalNormalized(), + UsageInKernelmode: taskStats.Stats.CPUStats.CPUUsage.UsageInKernelmode, + UsageInKernelmodePercentage: usage.InKernelMode(), + UsageInKernelmodePercentageNormalized: usage.InKernelModeNormalized(), + UsageInUsermode: taskStats.Stats.CPUStats.CPUUsage.UsageInUsermode, + UsageInUsermodePercentage: usage.InUserMode(), + UsageInUsermodePercentageNormalized: usage.InUserModeNormalized(), + SystemUsage: taskStats.Stats.CPUStats.SystemUsage, + SystemUsagePercentage: usage.System(), + SystemUsagePercentageNormalized: usage.SystemNormalized(), + } +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go new file mode 100644 index 00000000000..ca8cca5dca8 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -0,0 +1,164 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func eventsMapping(r mb.ReporterV2, statsList []Stats) { + for _, stats := range statsList { + r.Event(createEvent(&stats)) + } +} + +func createEvent(stats *Stats) mb.Event { + return mb.Event{ + Timestamp: time.Time(stats.Time), + RootFields: createContainerFields(stats), + MetricSetFields: common.MapStr{ + "cpu": createCPUFields(stats), + "memory": createMemoryFields(stats), + "network": createNetworkFields(stats), + "diskio": createDiskIOFields(stats), + }, + } +} + +func createContainerFields(stats *Stats) common.MapStr { + return common.MapStr{ + "container": common.MapStr{ + "id": stats.Container.DockerId, + "image": common.MapStr{ + "name": stats.Container.Image, + }, + "name": stats.Container.Name, + "labels": stats.Container.Labels, + }, + } +} + +func createCPUFields(stats *Stats) common.MapStr { + return common.MapStr{ + "core": stats.cpuStats.PerCPUUsage, + "total": common.MapStr{ + "pct": stats.cpuStats.TotalUsage, + "norm": common.MapStr{ + "pct": stats.cpuStats.TotalUsageNormalized, + }, + }, + "kernel": common.MapStr{ + "ticks": stats.cpuStats.UsageInKernelmode, + "pct": stats.cpuStats.UsageInKernelmodePercentage, + "norm": common.MapStr{ + "pct": stats.cpuStats.UsageInKernelmodePercentageNormalized, + }, + }, + "user": common.MapStr{ + "ticks": stats.cpuStats.UsageInUsermode, + "pct": stats.cpuStats.UsageInUsermodePercentage, + "norm": common.MapStr{ + "pct": stats.cpuStats.UsageInUsermodePercentageNormalized, + }, + }, + "system": common.MapStr{ + "ticks": stats.cpuStats.SystemUsage, + "pct": stats.cpuStats.SystemUsagePercentage, + "norm": common.MapStr{ + "pct": stats.cpuStats.SystemUsagePercentageNormalized, + }, + }, + } +} + +func createMemoryFields(stats *Stats) common.MapStr { + var memoryFields common.MapStr + if stats.memoryStats.Commit+stats.memoryStats.CommitPeak+stats.memoryStats.PrivateWorkingSet > 0 { + memoryFields = common.MapStr{ + "commit": common.MapStr{ + "total": stats.memoryStats.Commit, + "peak": stats.memoryStats.CommitPeak, + }, + "private_working_set": common.MapStr{ + "total": stats.memoryStats.PrivateWorkingSet, + }, + } + } else { + memoryFields = common.MapStr{ + "stats": stats.memoryStats.Stats, + "fail": common.MapStr{ + "count": stats.memoryStats.Failcnt, + }, + "limit": stats.memoryStats.Limit, + "rss": common.MapStr{ + "total": stats.memoryStats.TotalRss, + "pct": stats.memoryStats.TotalRssP, + }, + "usage": common.MapStr{ + "total": stats.memoryStats.Usage, + "pct": stats.memoryStats.UsageP, + "max": stats.memoryStats.MaxUsage, + }, + } + } + + return memoryFields +} + +func createNetworkFields(stats *Stats) common.MapStr { + networkFields := common.MapStr{} + for _, n := range stats.networkStats { + networkFields.Put(n.NameInterface, + common.MapStr{"inbound": common.MapStr{ + "bytes": n.Total.RxBytes, + "dropped": n.Total.RxDropped, + "errors": n.Total.RxErrors, + "packets": n.Total.RxPackets, + }, + "outbound": common.MapStr{ + "bytes": n.Total.TxBytes, + "dropped": n.Total.TxDropped, + "errors": n.Total.TxErrors, + "packets": n.Total.TxPackets, + }}) + } + return networkFields +} + +func createDiskIOFields(stats *Stats) common.MapStr { + return common.MapStr{ + "reads": stats.blkioStats.reads, + "writes": stats.blkioStats.writes, + "total": stats.blkioStats.totals, + "read": common.MapStr{ + "ops": stats.blkioStats.serviced.reads, + "bytes": stats.blkioStats.servicedBytes.reads, + "rate": stats.blkioStats.reads, + "service_time": stats.blkioStats.servicedTime.reads, + "wait_time": stats.blkioStats.waitTime.reads, + "queued": stats.blkioStats.queued.reads, + }, + "write": common.MapStr{ + "ops": stats.blkioStats.serviced.writes, + "bytes": stats.blkioStats.servicedBytes.writes, + "rate": stats.blkioStats.writes, + "service_time": stats.blkioStats.servicedTime.writes, + "wait_time": stats.blkioStats.waitTime.writes, + "queued": stats.blkioStats.queued.writes, + }, + "summary": common.MapStr{ + "ops": stats.blkioStats.serviced.totals, + "bytes": stats.blkioStats.servicedBytes.totals, + "rate": stats.blkioStats.totals, + "service_time": stats.blkioStats.servicedTime.totals, + "wait_time": stats.blkioStats.waitTime.totals, + "queued": stats.blkioStats.queued.totals, + }, + } + +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/diskio.go b/x-pack/metricbeat/module/awsfargate/task_stats/diskio.go new file mode 100644 index 00000000000..c7d4791aa48 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/diskio.go @@ -0,0 +1,57 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import "github.com/docker/docker/api/types" + +// BlkioRaw sums raw Blkio stats +type BlkioRaw struct { + reads uint64 + writes uint64 + totals uint64 +} + +type blkioStats struct { + reads float64 + writes float64 + totals float64 + + serviced BlkioRaw + servicedBytes BlkioRaw + servicedTime BlkioRaw + waitTime BlkioRaw + queued BlkioRaw +} + +// getBlkioStats collects diskio metrics from BlkioStats structures(not populated in Windows) +func getBlkioStats(raw types.BlkioStats) blkioStats { + return blkioStats{ + serviced: getNewStats(raw.IoServicedRecursive), + servicedBytes: getNewStats(raw.IoServiceBytesRecursive), + servicedTime: getNewStats(raw.IoServiceTimeRecursive), + waitTime: getNewStats(raw.IoWaitTimeRecursive), + queued: getNewStats(raw.IoQueuedRecursive), + } +} + +func getNewStats(blkioEntry []types.BlkioStatEntry) BlkioRaw { + stats := BlkioRaw{ + reads: 0, + writes: 0, + totals: 0, + } + + for _, myEntry := range blkioEntry { + switch myEntry.Op { + case "Write": + stats.writes += myEntry.Value + case "Read": + stats.reads += myEntry.Value + case "Total": + stats.totals += myEntry.Value + } + } + return stats +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/memory.go b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go new file mode 100644 index 00000000000..0dfb68d9317 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go @@ -0,0 +1,40 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import "github.com/docker/docker/api/types" + +type memoryStats struct { + Failcnt uint64 + Limit uint64 + MaxUsage uint64 + TotalRss uint64 + TotalRssP float64 + Usage uint64 + UsageP float64 + //Raw stats from the cgroup subsystem + Stats map[string]uint64 + //Windows-only memory stats + Commit uint64 + CommitPeak uint64 + PrivateWorkingSet uint64 +} + +func getMemoryStats(taskStats types.StatsJSON) memoryStats { + totalRSS := taskStats.Stats.MemoryStats.Stats["total_rss"] + + return memoryStats{ + TotalRss: totalRSS, + MaxUsage: taskStats.Stats.MemoryStats.MaxUsage, + TotalRssP: float64(totalRSS) / float64(taskStats.Stats.MemoryStats.Limit), + Usage: taskStats.Stats.MemoryStats.Usage, + UsageP: float64(taskStats.Stats.MemoryStats.Usage) / float64(taskStats.Stats.MemoryStats.Limit), + Stats: taskStats.Stats.MemoryStats.Stats, + //Windows memory statistics + Commit: taskStats.Stats.MemoryStats.Commit, + CommitPeak: taskStats.Stats.MemoryStats.CommitPeak, + PrivateWorkingSet: taskStats.Stats.MemoryStats.PrivateWorkingSet, + } +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/network.go b/x-pack/metricbeat/module/awsfargate/task_stats/network.go new file mode 100644 index 00000000000..80cec79748a --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/network.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import "github.com/docker/docker/api/types" + +type networkStats struct { + NameInterface string + Total types.NetworkStats +} + +func getNetworkStats(taskStats types.StatsJSON) []networkStats { + var networks []networkStats + for nameInterface, rawNetStats := range taskStats.Networks { + networks = append(networks, networkStats{ + NameInterface: nameInterface, + Total: rawNetStats, + }) + } + return networks +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go new file mode 100644 index 00000000000..d4e8316611b --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -0,0 +1,193 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + + "github.com/docker/docker/api/types" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/docker/cpu" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/awsfargate" +) + +var ( + metricsetName = "task_stats" + taskStatsPath = "task/stats" + taskPath = "task" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(awsfargate.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *awsfargate.MetricSet + logger *logp.Logger + taskStatsEndpoint string + taskEndpoint string +} + +// Stats is a struct represents information regarding a container +type Stats struct { + Time common.Time + Container *container + cpuStats cpu.CPUStats + memoryStats memoryStats + networkStats []networkStats + blkioStats blkioStats +} + +// TaskMetadata is an struct represents response body from ${ECS_CONTAINER_METADATA_URI_V4}/task +type TaskMetadata struct { + Cluster string `json:"Cluster"` + TaskARN string `json:"TaskARN"` + Family string `json:"Family"` + Revision string `json:"Revision"` + Containers []*container `json:"Containers"` +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logger := logp.NewLogger(metricsetName) + metricSet, err := awsfargate.NewMetricSet(base) + if err != nil { + return nil, fmt.Errorf("error creating %s metricset: %w", metricsetName, err) + } + + ecsURI, ok := os.LookupEnv("ECS_CONTAINER_METADATA_URI_V4") + if !ok { + return nil, fmt.Errorf("lookup $ECS_CONTAINER_METADATA_URI_V4 failed") + } + + return &MetricSet{ + MetricSet: metricSet, + logger: logger, + taskStatsEndpoint: fmt.Sprintf("%s/%s", ecsURI, taskStatsPath), + taskEndpoint: fmt.Sprintf("%s/%s", ecsURI, taskPath), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + formattedStats, err := m.queryTaskMetadataEndpoints() + if err != nil { + err := fmt.Errorf("queryTaskMetadataEndpoints failed: %w", err) + m.logger.Error(err) + return err + } + + eventsMapping(report, formattedStats) + return nil +} + +func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { + // Get response from ${ECS_CONTAINER_METADATA_URI_V4}/task/stats + taskStatsResp, err := http.Get(m.taskStatsEndpoint) + if err != nil { + return nil, fmt.Errorf("http.Get failed: %w", err) + } + taskStatsOutput, err := getTaskStats(taskStatsResp) + if err != nil { + return nil, fmt.Errorf("getTaskStats failed: %w", err) + } + + // Collect container metadata information from ${ECS_CONTAINER_METADATA_URI_V4}/task + taskResp, err := http.Get(m.taskEndpoint) + if err != nil { + return nil, fmt.Errorf("http.Get failed: %w", err) + } + taskOutput, err := getTask(taskResp) + if err != nil { + return nil, fmt.Errorf("getTask failed: %w", err) + } + + formattedStats := getStatsList(taskStatsOutput, taskOutput) + return formattedStats, nil +} + +func getTaskStats(taskStatsResp *http.Response) (map[string]types.StatsJSON, error) { + taskStatsBody, err := ioutil.ReadAll(taskStatsResp.Body) + if err != nil { + return nil, fmt.Errorf("ioutil.ReadAll failed: %w", err) + } + + var taskStatsOutput map[string]types.StatsJSON + err = json.Unmarshal(taskStatsBody, &taskStatsOutput) + if err != nil { + return nil, fmt.Errorf("json.Unmarshal failed: %w", err) + } + return taskStatsOutput, nil +} + +func getTask(taskResp *http.Response) (TaskMetadata, error) { + taskBody, err := ioutil.ReadAll(taskResp.Body) + if err != nil { + return TaskMetadata{}, fmt.Errorf("ioutil.ReadAll failed: %w", err) + } + + var taskOutput TaskMetadata + err = json.Unmarshal(taskBody, &taskOutput) + if err != nil { + return TaskMetadata{}, fmt.Errorf("json.Unmarshal failed: %w", err) + } + return taskOutput, nil +} + +func getStatsList(taskStatsOutput map[string]types.StatsJSON, taskOutput TaskMetadata) []Stats { + containersInfo := map[string]ContainerMetadata{} + for _, c := range taskOutput.Containers { + // Skip ~internal~ecs~pause container + if c.Name == "~internal~ecs~pause" { + continue + } + + containerMetadata := ContainerMetadata{ + Container: c, + Family: taskOutput.Family, + TaskARN: taskOutput.TaskARN, + Cluster: taskOutput.Cluster, + Revision: taskOutput.Revision, + } + containersInfo[c.DockerId] = containerMetadata + } + + var formattedStats []Stats + for id, taskStats := range taskStatsOutput { + if cInfo, ok := containersInfo[id]; ok { + statsPerContainer := Stats{ + Time: common.Time(taskStats.Stats.Read), + Container: getContainerStats(cInfo.Container), + cpuStats: getCPUStats(taskStats), + memoryStats: getMemoryStats(taskStats), + networkStats: getNetworkStats(taskStats), + blkioStats: getBlkioStats(taskStats.BlkioStats), + } + + formattedStats = append(formattedStats, statsPerContainer) + } + } + return formattedStats +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go new file mode 100644 index 00000000000..7193f640a63 --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -0,0 +1,69 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration + +package task_stats + +import ( + "bytes" + "io/ioutil" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/mb/testing/flags" +) + +func TestData(t *testing.T) { + if !*flags.DataFlag { + t.Skip("skip data generation tests") + } + + config := map[string]interface{}{ + "period": "10s", + "module": "awsfargate", + "metricsets": []string{"task_stats"}, + } + + m := mbtest.NewFetcher(t, config) + + taskStatsFile, err := os.Open("./_meta/testdata/task_stats.json") + assert.NoError(t, err) + defer taskStatsFile.Close() + + byteTaskStats, err := ioutil.ReadAll(taskStatsFile) + assert.NoError(t, err) + + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader(byteTaskStats)), + } + + taskFile, err := os.Open("./_meta/testdata/task.json") + assert.NoError(t, err) + defer taskStatsFile.Close() + + byteTask, err := ioutil.ReadAll(taskFile) + assert.NoError(t, err) + + byteTaskResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader(byteTask)), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + + taskOutput, err := getTask(byteTaskResp) + assert.NoError(t, err) + + formattedStats := getStatsList(taskStatsOutput, taskOutput) + assert.Equal(t, 1, len(formattedStats)) + event := createEvent(&formattedStats[0]) + standardizeEvent := m.StandardizeEvent(event) + + mbtest.WriteEventToDataJSON(t, standardizeEvent, "") +} diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go new file mode 100644 index 00000000000..c54c3d5efdb --- /dev/null +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package task_stats + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +var ( + taskStatsJson = `{ + "query-metadata-1": { + "read": "2020-04-06T16:12:01.090148907Z", + "preread": "2020-04-06T16:12:01.090148907Z", + "cpu_stats": { + "cpu_usage": { + "percpu_usage": [1800000000, 500000000, 0, 0, 0, 0, 0, 0], + "total_usage": 2300000000, "usage_in_kernelmode": 1520000000, "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1420180000000, + "throttling_data": {"periods": 0, "throttled_periods": 0, "throttled_time": 0}}, + "precpu_stats": { + "cpu_usage": { + "percpu_usage": [1600000000, 300000000, 0, 0, 0, 0, 0, 0], + "total_usage": 1900000000, "usage_in_kernelmode": 1520000000, "usage_in_usermode": 490000000 + }, + "online_cpus": 2, + "system_cpu_usage": 1418180000000, + "throttling_data": {"periods": 0, "throttled_periods": 0, "throttled_time": 0}}, + "memory_stats": {"limit": 8362348544, "usage": 4390912, "max_usage": 6488064, "stats": {"total_rss": 278528}}, + "name": "query-metadata-1", + "id": "query-metadata-1", + "networks": {"eth0": {"rx_bytes": 1802, "rx_packets": 19, "rx_errors": 0, "rx_dropped": 0, + "tx_bytes": 567, "tx_packets": 7, "tx_errors": 0, "tx_dropped": 0}} + }}` + + taskRespJson = `{ + "Cluster": "arn:aws:ecs:us-west-2:123:cluster/default", + "TaskARN": "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", + "Family": "query-metadata-1", + "Revision": "7", + "Containers": [{ + "DockerId": "query-metadata-1", + "Name": "query-metadata", + "Image": "mreferre/eksutils", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "query-metadata", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", + "com.amazonaws.ecs.task-definition-family": "query-metadata", + "com.amazonaws.ecs.task-definition-version": "7"} + }] + }` +) + +func TestGetTaskStats(t *testing.T) { + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + assert.Equal(t, uint64(2300000000), taskStatsOutput["query-metadata-1"].CPUStats.CPUUsage.TotalUsage) +} + +func TestGetTask(t *testing.T) { + taskResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + } + + taskOutput, err := getTask(taskResp) + assert.NoError(t, err) + + assert.Equal(t, "arn:aws:ecs:us-west-2:123:cluster/default", taskOutput.Cluster) + assert.Equal(t, "arn:aws:ecs:us-west-2:123:task/default/febee207c04a", taskOutput.TaskARN) + assert.Equal(t, "query-metadata-1", taskOutput.Family) + assert.Equal(t, "7", taskOutput.Revision) + + assert.Equal(t, 1, len(taskOutput.Containers)) + assert.Equal(t, "query-metadata-1", taskOutput.Containers[0].DockerId) + assert.Equal(t, "query-metadata", taskOutput.Containers[0].Name) + assert.Equal(t, "mreferre/eksutils", taskOutput.Containers[0].Image) + assert.Equal(t, 5, len(taskOutput.Containers[0].Labels)) +} + +func TestGetStatsList(t *testing.T) { + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + + taskResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + } + + taskOutput, err := getTask(taskResp) + assert.NoError(t, err) + + formattedStats := getStatsList(taskStatsOutput, taskOutput) + assert.Equal(t, 1, len(formattedStats)) +} + +func TestGetCPUStats(t *testing.T) { + taskStatsResp := &http.Response{ + Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + } + + taskStatsOutput, err := getTaskStats(taskStatsResp) + assert.NoError(t, err) + assert.Equal(t, 1, len(taskStatsOutput)) + + cpuStats := getCPUStats(taskStatsOutput["query-metadata-1"]) + assert.Equal(t, 0.4, cpuStats.TotalUsage) + assert.Equal(t, 0.2, cpuStats.TotalUsageNormalized) +} diff --git a/x-pack/metricbeat/modules.d/awsfargate.yml.disabled b/x-pack/metricbeat/modules.d/awsfargate.yml.disabled new file mode 100644 index 00000000000..b2b91f06ee4 --- /dev/null +++ b/x-pack/metricbeat/modules.d/awsfargate.yml.disabled @@ -0,0 +1,7 @@ +# Module: awsfargate +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-awsfargate.html + +- module: awsfargate + period: 10s + metricsets: + - task_stats