Add system/raid metricset to Metricbeat

Closes #5600

CHANGELOG.asciidoc

@@ -135,6 +135,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add experimental Docker autodiscover functionality. {pull}5245[5245]
 - Add field network_names of hosts and virtual machines. {issue}5646[5646]
 - Update gosigar to v0.6.0. {pull}5775[5775]
+- Add experimental system/raid metricset. {pull}5642[5642]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -11187,6 +11187,61 @@ type: long
 Number of processes for which the state couldn't be retrieved or is unknown.
 
 
+[float]
+== raid fields
+
+raid
+
+
+
+[float]
+=== `system.raid.name`
+
+type: keyword
+
+Name of the device.
+
+
+[float]
+=== `system.raid.activity_state`
+
+type: keyword
+
+activity-state of the device.
+
+
+[float]
+=== `system.raid.disks.active`
+
+type: long
+
+Number of active disks.
+
+
+[float]
+=== `system.raid.disks.total`
+
+type: long
+
+Total number of disks the device consists of.
+
+
+[float]
+=== `system.raid.blocks.total`
+
+type: long
+
+Number of blocks the device holds.
+
+
+[float]
+=== `system.raid.blocks.synced`
+
+type: long
+
+Number of blocks on the device that are in sync.
+
+
 [float]
 == socket fields
 

metricbeat/docs/modules/system.asciidoc

@@ -82,6 +82,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-system-process_summary,process_summary>>
 
+* <<metricbeat-metricset-system-raid,raid>>
+
 * <<metricbeat-metricset-system-socket,socket>>
 
 * <<metricbeat-metricset-system-uptime,uptime>>
@@ -106,6 +108,8 @@ include::system/process.asciidoc[]
 
 include::system/process_summary.asciidoc[]
 
+include::system/raid.asciidoc[]
+
 include::system/socket.asciidoc[]
 
 include::system/uptime.asciidoc[]

metricbeat/docs/modules/system/raid.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-system-raid]]
+=== System raid metricset
+
+experimental[]
+
+include::../../../module/system/raid/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-system,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/system/raid/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -97,7 +97,7 @@ This file is generated! See scripts/docs_collector.py
 .2+|   |<<metricbeat-metricset-redis-info,info>>   
 |<<metricbeat-metricset-redis-keyspace,keyspace>>   
 |<<metricbeat-module-system,System>>     |    
-.12+|   |<<metricbeat-metricset-system-core,core>>   
+.13+|   |<<metricbeat-metricset-system-core,core>>   
 |<<metricbeat-metricset-system-cpu,cpu>>   
 |<<metricbeat-metricset-system-diskio,diskio>>   
 |<<metricbeat-metricset-system-filesystem,filesystem>>   
@@ -107,6 +107,7 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-metricset-system-network,network>>   
 |<<metricbeat-metricset-system-process,process>>   
 |<<metricbeat-metricset-system-process_summary,process_summary>>   
+|<<metricbeat-metricset-system-raid,raid>> experimental[]  
 |<<metricbeat-metricset-system-socket,socket>> beta[]  
 |<<metricbeat-metricset-system-uptime,uptime>>   
 |<<metricbeat-module-vsphere,vSphere>>  beta[]   |    

metricbeat/include/list.go

@@ -111,6 +111,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/system/network"
 	_ "github.com/elastic/beats/metricbeat/module/system/process"
 	_ "github.com/elastic/beats/metricbeat/module/system/process_summary"
+	_ "github.com/elastic/beats/metricbeat/module/system/raid"
 	_ "github.com/elastic/beats/metricbeat/module/system/socket"
 	_ "github.com/elastic/beats/metricbeat/module/system/uptime"
 	_ "github.com/elastic/beats/metricbeat/module/vsphere"

metricbeat/module/system/raid/_meta/data.json

@@ -0,0 +1,26 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "metricset": {
+        "module": "system",
+        "name": "raid",
+        "rtt": 115
+    },
+    "system": {
+        "raid": {
+            "activity_state": "active",
+            "blocks": {
+                "synced": 5853468288,
+                "total": 5853468288
+            },
+            "disks": {
+                "active": 8,
+                "total": 8
+            },
+            "name": "md3"
+        }
+    }
+}

metricbeat/module/system/raid/_meta/docs.asciidoc

@@ -0,0 +1,3 @@
+=== system raid MetricSet
+
+This is the raid metricset of the module system.

metricbeat/module/system/raid/_meta/fields.yml

@@ -0,0 +1,30 @@
+- name: raid
+  type: group
+  description: >
+    raid
+  release: experimental
+  fields:
+    - name: name
+      type: keyword
+      description: >
+        Name of the device.
+    - name: activity_state
+      type: keyword
+      description: >
+        activity-state of the device.
+    - name: disks.active
+      type: long
+      description: >
+         Number of active disks.
+    - name: disks.total
+      type: long
+      description: >
+        Total number of disks the device consists of.
+    - name: blocks.total
+      type: long
+      description: >
+        Number of blocks the device holds.
+    - name: blocks.synced
+      type: long
+      description: >
+        Number of blocks on the device that are in sync.

metricbeat/module/system/raid/_meta/testdata/proc/mdstat

@@ -0,0 +1,26 @@
+Personalities : [raid0] [raid1] [linear] [multipath] [raid6] [raid5] [raid4] [raid10]
+md3 : active raid6 sda1[8] sdh1[7] sdg1[6] sdf1[5] sde1[11] sdd1[3] sdc1[10] sdb1[9]
+      5853468288 blocks super 1.2 level 6, 64k chunk, algorithm 2 [8/8] [UUUUUUUU]
+
+md127 : active raid1 sdi2[0] sdj2[1]
+      312319552 blocks [2/2] [UU]
+
+md0 : active raid1 sdk[2](S) sdi1[0] sdj1[1]
+      248896 blocks [2/2] [UU]
+
+md4 : inactive raid1 sda3[0] sdb3[1]
+      4883648 blocks [2/2] [UU]
+
+md6 : active raid1 sdb2[2] sda2[0]
+      195310144 blocks [2/1] [U_]
+      [=>...................]  recovery =  8.5% (16775552/195310144) finish=17.0min speed=259783K/sec
+
+md8 : active raid1 sdb1[1] sda1[0]
+      195310144 blocks [2/2] [UU]
+      [=>...................]  resync =  8.5% (16775552/195310144) finish=17.0min speed=259783K/sec
+
+md7 : active raid6 sdb1[0] sde1[3] sdd1[2] sdc1[1]
+      7813735424 blocks super 1.2 level 6, 512k chunk, algorithm 2 [4/3] [U_UU]
+      bitmap: 0/30 pages [0KB], 65536KB chunk
+
+unused devices: <none>

metricbeat/module/system/raid/doc.go

@@ -0,0 +1 @@
+package raid

metricbeat/module/system/raid/raid.go

@@ -0,0 +1,90 @@
+package raid
+
+import (
+	"path/filepath"
+
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/cfgwarn"
+	"github.com/elastic/beats/metricbeat/mb"
+	"github.com/elastic/beats/metricbeat/mb/parse"
+	"github.com/elastic/beats/metricbeat/module/system"
+	"github.com/elastic/procfs"
+
+	"github.com/pkg/errors"
+)
+
+func init() {
+	if err := mb.Registry.AddMetricSet("system", "raid", New, parse.EmptyHostParser); err != nil {
+		panic(err)
+	}
+}
+
+// MetricSet contains proc fs data.
+type MetricSet struct {
+	mb.BaseMetricSet
+	fs procfs.FS
+}
+
+// New creates a new instance of the raid metricset.
+func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	cfgwarn.Experimental("The system raid metricset is experimental")
+
+	systemModule, ok := base.Module().(*system.Module)
+	if !ok {
+		return nil, errors.New("unexpected module type")
+	}
+
+	// Additional configuration options
+	config := struct {
+		MountPoint string `config:"raid.mount_point"`
+	}{}
+
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
+
+	if config.MountPoint == "" {
+		config.MountPoint = systemModule.HostFS
+	}
+
+	mountPoint := filepath.Join(config.MountPoint, procfs.DefaultMountPoint)
+	fs, err := procfs.NewFS(mountPoint)
+	if err != nil {
+		return nil, err
+	}
+
+	m := &MetricSet{
+		BaseMetricSet: base,
+		fs:            fs,
+	}
+
+	return m, nil
+}
+
+// Fetch fetches one event for each device
+func (m *MetricSet) Fetch() ([]common.MapStr, error) {
+
+	stats, err := m.fs.ParseMDStat()
+	if err != nil {
+		return nil, err
+	}
+
+	events := make([]common.MapStr, 0, len(stats))
+	for _, stat := range stats {
+		event := common.MapStr{
+			"name":           stat.Name,
+			"activity_state": stat.ActivityState,
+			"disks": common.MapStr{
+				"active": stat.DisksActive,
+				"total":  stat.DisksTotal,
+			},
+			"blocks": common.MapStr{
+				"synced": stat.BlocksSynced,
+				"total":  stat.BlocksTotal,
+			},
+		}
+		events = append(events, event)
+	}
+
+	return events, nil
+}

metricbeat/module/system/raid/raid_test.go

@@ -0,0 +1,32 @@
+package raid
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+)
+
+func TestData(t *testing.T) {
+	f := mbtest.NewEventsFetcher(t, getConfig())
+
+	if err := mbtest.WriteEvents(f, t); err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func TestFetch(t *testing.T) {
+	f := mbtest.NewEventsFetcher(t, getConfig())
+	data, err := f.Fetch()
+	assert.NoError(t, err)
+	assert.Equal(t, 7, len(data))
+}
+
+func getConfig() map[string]interface{} {
+	return map[string]interface{}{
+		"module":           "system",
+		"metricsets":       []string{"raid"},
+		"raid.mount_point": "./_meta/testdata",
+	}
+}