From 46c2691470efebe1579fbd2e525850abc0ee569e Mon Sep 17 00:00:00 2001 From: ruflin Date: Mon, 20 Nov 2017 14:53:27 +1100 Subject: [PATCH] Add system/raid metricset to Metricbeat Closes https://github.com/elastic/beats/issues/5600 --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 55 ++++++++++++ metricbeat/docs/modules/system.asciidoc | 4 + metricbeat/docs/modules/system/raid.asciidoc | 23 +++++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 1 + metricbeat/module/system/raid/_meta/data.json | 26 ++++++ .../module/system/raid/_meta/docs.asciidoc | 3 + .../module/system/raid/_meta/fields.yml | 30 +++++++ .../system/raid/_meta/testdata/proc/mdstat | 26 ++++++ metricbeat/module/system/raid/doc.go | 1 + metricbeat/module/system/raid/raid.go | 90 +++++++++++++++++++ metricbeat/module/system/raid/raid_test.go | 32 +++++++ 13 files changed, 294 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/modules/system/raid.asciidoc create mode 100644 metricbeat/module/system/raid/_meta/data.json create mode 100644 metricbeat/module/system/raid/_meta/docs.asciidoc create mode 100644 metricbeat/module/system/raid/_meta/fields.yml create mode 100644 metricbeat/module/system/raid/_meta/testdata/proc/mdstat create mode 100644 metricbeat/module/system/raid/doc.go create mode 100644 metricbeat/module/system/raid/raid.go create mode 100644 metricbeat/module/system/raid/raid_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index db00670ea46..b433f35770f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -135,6 +135,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Add experimental Docker autodiscover functionality. {pull}5245[5245] - Add field network_names of hosts and virtual machines. {issue}5646[5646] - Update gosigar to v0.6.0. {pull}5775[5775] +- Add experimental system/raid metricset. {pull}5642[5642] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 0328e579718..8a2521430ab 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -11187,6 +11187,61 @@ type: long Number of processes for which the state couldn't be retrieved or is unknown. +[float] +== raid fields + +raid + + + +[float] +=== `system.raid.name` + +type: keyword + +Name of the device. + + +[float] +=== `system.raid.activity_state` + +type: keyword + +activity-state of the device. + + +[float] +=== `system.raid.disks.active` + +type: long + +Number of active disks. + + +[float] +=== `system.raid.disks.total` + +type: long + +Total number of disks the device consists of. + + +[float] +=== `system.raid.blocks.total` + +type: long + +Number of blocks the device holds. + + +[float] +=== `system.raid.blocks.synced` + +type: long + +Number of blocks on the device that are in sync. + + [float] == socket fields diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc index 7f3c48f776e..56f36d6cde0 100644 --- a/metricbeat/docs/modules/system.asciidoc +++ b/metricbeat/docs/modules/system.asciidoc @@ -82,6 +82,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -106,6 +108,8 @@ include::system/process.asciidoc[] include::system/process_summary.asciidoc[] +include::system/raid.asciidoc[] + include::system/socket.asciidoc[] include::system/uptime.asciidoc[] diff --git a/metricbeat/docs/modules/system/raid.asciidoc b/metricbeat/docs/modules/system/raid.asciidoc new file mode 100644 index 00000000000..f48c0b2a867 --- /dev/null +++ b/metricbeat/docs/modules/system/raid.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-system-raid]] +=== System raid metricset + +experimental[] + +include::../../../module/system/raid/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/system/raid/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 09210491ba1..cb0dca18ae5 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -97,7 +97,7 @@ This file is generated! See scripts/docs_collector.py .2+| |<> |<> |<> | -.12+| |<> +.13+| |<> |<> |<> |<> @@ -107,6 +107,7 @@ This file is generated! See scripts/docs_collector.py |<> |<> |<> +|<> experimental[] |<> beta[] |<> |<> beta[] | diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index c6403c5258e..1596e55bb86 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -111,6 +111,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/system/network" _ "github.com/elastic/beats/metricbeat/module/system/process" _ "github.com/elastic/beats/metricbeat/module/system/process_summary" + _ "github.com/elastic/beats/metricbeat/module/system/raid" _ "github.com/elastic/beats/metricbeat/module/system/socket" _ "github.com/elastic/beats/metricbeat/module/system/uptime" _ "github.com/elastic/beats/metricbeat/module/vsphere" diff --git a/metricbeat/module/system/raid/_meta/data.json b/metricbeat/module/system/raid/_meta/data.json new file mode 100644 index 00000000000..b0ea391ccf5 --- /dev/null +++ b/metricbeat/module/system/raid/_meta/data.json @@ -0,0 +1,26 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "module": "system", + "name": "raid", + "rtt": 115 + }, + "system": { + "raid": { + "activity_state": "active", + "blocks": { + "synced": 5853468288, + "total": 5853468288 + }, + "disks": { + "active": 8, + "total": 8 + }, + "name": "md3" + } + } +} \ No newline at end of file diff --git a/metricbeat/module/system/raid/_meta/docs.asciidoc b/metricbeat/module/system/raid/_meta/docs.asciidoc new file mode 100644 index 00000000000..6544b9da599 --- /dev/null +++ b/metricbeat/module/system/raid/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== system raid MetricSet + +This is the raid metricset of the module system. diff --git a/metricbeat/module/system/raid/_meta/fields.yml b/metricbeat/module/system/raid/_meta/fields.yml new file mode 100644 index 00000000000..405c04c0586 --- /dev/null +++ b/metricbeat/module/system/raid/_meta/fields.yml @@ -0,0 +1,30 @@ +- name: raid + type: group + description: > + raid + release: experimental + fields: + - name: name + type: keyword + description: > + Name of the device. + - name: activity_state + type: keyword + description: > + activity-state of the device. + - name: disks.active + type: long + description: > + Number of active disks. + - name: disks.total + type: long + description: > + Total number of disks the device consists of. + - name: blocks.total + type: long + description: > + Number of blocks the device holds. + - name: blocks.synced + type: long + description: > + Number of blocks on the device that are in sync. diff --git a/metricbeat/module/system/raid/_meta/testdata/proc/mdstat b/metricbeat/module/system/raid/_meta/testdata/proc/mdstat new file mode 100644 index 00000000000..162aa1b984f --- /dev/null +++ b/metricbeat/module/system/raid/_meta/testdata/proc/mdstat @@ -0,0 +1,26 @@ +Personalities : [raid0] [raid1] [linear] [multipath] [raid6] [raid5] [raid4] [raid10] +md3 : active raid6 sda1[8] sdh1[7] sdg1[6] sdf1[5] sde1[11] sdd1[3] sdc1[10] sdb1[9] + 5853468288 blocks super 1.2 level 6, 64k chunk, algorithm 2 [8/8] [UUUUUUUU] + +md127 : active raid1 sdi2[0] sdj2[1] + 312319552 blocks [2/2] [UU] + +md0 : active raid1 sdk[2](S) sdi1[0] sdj1[1] + 248896 blocks [2/2] [UU] + +md4 : inactive raid1 sda3[0] sdb3[1] + 4883648 blocks [2/2] [UU] + +md6 : active raid1 sdb2[2] sda2[0] + 195310144 blocks [2/1] [U_] + [=>...................] recovery = 8.5% (16775552/195310144) finish=17.0min speed=259783K/sec + +md8 : active raid1 sdb1[1] sda1[0] + 195310144 blocks [2/2] [UU] + [=>...................] resync = 8.5% (16775552/195310144) finish=17.0min speed=259783K/sec + +md7 : active raid6 sdb1[0] sde1[3] sdd1[2] sdc1[1] + 7813735424 blocks super 1.2 level 6, 512k chunk, algorithm 2 [4/3] [U_UU] + bitmap: 0/30 pages [0KB], 65536KB chunk + +unused devices: diff --git a/metricbeat/module/system/raid/doc.go b/metricbeat/module/system/raid/doc.go new file mode 100644 index 00000000000..7307d0144ba --- /dev/null +++ b/metricbeat/module/system/raid/doc.go @@ -0,0 +1 @@ +package raid diff --git a/metricbeat/module/system/raid/raid.go b/metricbeat/module/system/raid/raid.go new file mode 100644 index 00000000000..23eae508ed7 --- /dev/null +++ b/metricbeat/module/system/raid/raid.go @@ -0,0 +1,90 @@ +package raid + +import ( + "path/filepath" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/system" + "github.com/elastic/procfs" + + "github.com/pkg/errors" +) + +func init() { + if err := mb.Registry.AddMetricSet("system", "raid", New, parse.EmptyHostParser); err != nil { + panic(err) + } +} + +// MetricSet contains proc fs data. +type MetricSet struct { + mb.BaseMetricSet + fs procfs.FS +} + +// New creates a new instance of the raid metricset. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The system raid metricset is experimental") + + systemModule, ok := base.Module().(*system.Module) + if !ok { + return nil, errors.New("unexpected module type") + } + + // Additional configuration options + config := struct { + MountPoint string `config:"raid.mount_point"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + if config.MountPoint == "" { + config.MountPoint = systemModule.HostFS + } + + mountPoint := filepath.Join(config.MountPoint, procfs.DefaultMountPoint) + fs, err := procfs.NewFS(mountPoint) + if err != nil { + return nil, err + } + + m := &MetricSet{ + BaseMetricSet: base, + fs: fs, + } + + return m, nil +} + +// Fetch fetches one event for each device +func (m *MetricSet) Fetch() ([]common.MapStr, error) { + + stats, err := m.fs.ParseMDStat() + if err != nil { + return nil, err + } + + events := make([]common.MapStr, 0, len(stats)) + for _, stat := range stats { + event := common.MapStr{ + "name": stat.Name, + "activity_state": stat.ActivityState, + "disks": common.MapStr{ + "active": stat.DisksActive, + "total": stat.DisksTotal, + }, + "blocks": common.MapStr{ + "synced": stat.BlocksSynced, + "total": stat.BlocksTotal, + }, + } + events = append(events, event) + } + + return events, nil +} diff --git a/metricbeat/module/system/raid/raid_test.go b/metricbeat/module/system/raid/raid_test.go new file mode 100644 index 00000000000..bb5fb15027b --- /dev/null +++ b/metricbeat/module/system/raid/raid_test.go @@ -0,0 +1,32 @@ +package raid + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + + if err := mbtest.WriteEvents(f, t); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + f := mbtest.NewEventsFetcher(t, getConfig()) + data, err := f.Fetch() + assert.NoError(t, err) + assert.Equal(t, 7, len(data)) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "system", + "metricsets": []string{"raid"}, + "raid.mount_point": "./_meta/testdata", + } +}