Index metricset for elasticsearch Metricbeat module (#6881)

* Index metricset for elasticsearch Metricbeat module

This adds the index metricset to the Elasticsearch module. For now it only contains very basic metrics and is marked experimental.

This PR is used to create the basic for adding tests with the fetch reporter v2 interface and also generate data with it.

It also adds basic utility functions to fetch Master or Node information in the Elasticsearch module.

CHANGELOG.asciidoc

@@ -128,7 +128,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add MongoDB module. {pull}6283[6238]
 - Add Ingest pipeline loading to setup. {pull}6814[6814]
 - Add support of log_format combined to NGINX access logs. {pull}6858[6858]
-- Release config reloading feature as GA.
+- Release config reloading feature as GA. {pull}6891[6891]
 - Add support human friendly size for the UDP input. {pull}6886[6886]
 - Add Syslog input to ingest RFC3164 Events via TCP and UDP {pull}6842[6842]
 
@@ -199,7 +199,8 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Added support for haproxy 1.7 and 1.8. {pull}6793[6793]
 - Add accumulated I/O stats to diskio in the line of `docker stats`. {pull}6701[6701]
 - Ignore virtual filesystem types by default in system module. {pull}6819[6819]
-- Release config reloading feature as GA.
+- Release config reloading feature as GA. {pull}6891[6891]
+- Add experimental Elasticsearch index metricset. {pull}6881[6881]
 
 *Packetbeat*
 

metricbeat/_meta/fields.common.yml

@@ -34,3 +34,7 @@
       description: >
         The document type. Always set to "doc".
 
+    - name: service.name
+      example: elasticsearch
+      description: >
+        Name of the service metricbeat fetches the data from.

metricbeat/docs/fields.asciidoc

@@ -1862,6 +1862,16 @@ required: True
 The document type. Always set to "doc".
 
 
+--
+
+*`service.name`*::
++
+--
+example: elasticsearch
+
+Name of the service metricbeat fetches the data from.
+
+
 --
 
 [[exported-fields-couchbase]]
@@ -3317,6 +3327,88 @@ type: keyword
 Elasticsearch cluster name.
 
 
+--
+
+*`elasticsearch.cluster.id`*::
++
+--
+type: keyword
+
+Elasticsearch cluster id.
+
+
+--
+
+[float]
+== index fields
+
+index
+
+
+
+*`elasticsearch.index.name`*::
++
+--
+type: keyword
+
+Index name.
+
+
+--
+
+
+*`elasticsearch.index.total.docs.count`*::
++
+--
+type: long
+
+Total number of documents in the index.
+
+
+--
+
+*`elasticsearch.index.total.docs.deleted`*::
++
+--
+type: long
+
+Total number of deleted documents in the index.
+
+
+--
+
+*`elasticsearch.index.total.store.size.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total size of the index in bytes.
+
+
+--
+
+*`elasticsearch.index.total.segments.count`*::
++
+--
+type: long
+
+Total number of index segments.
+
+
+--
+
+*`elasticsearch.index.total.segments.memory.bytes`*::
++
+--
+type: long
+
+format: bytes
+
+Total number of memory used by the segments in bytes.
+
+
 --
 
 [float]

metricbeat/docs/modules/elasticsearch.asciidoc

@@ -22,7 +22,10 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: elasticsearch
-  metricsets: ["node", "node_stats"]
+  metricsets:
+   #- index
+   - node
+   - node_stats
   period: 10s
   hosts: ["localhost:9200"]
 ----
@@ -34,10 +37,14 @@ This module supports TLS connection when using `ssl` config field, as described
 
 The following metricsets are available:
 
+* <<metricbeat-metricset-elasticsearch-index,index>>
+
 * <<metricbeat-metricset-elasticsearch-node,node>>
 
 * <<metricbeat-metricset-elasticsearch-node_stats,node_stats>>
 
+include::elasticsearch/index.asciidoc[]
+
 include::elasticsearch/node.asciidoc[]
 
 include::elasticsearch/node_stats.asciidoc[]

metricbeat/docs/modules/elasticsearch/index.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-elasticsearch-index]]
+=== Elasticsearch index metricset
+
+experimental[]
+
+include::../../../module/elasticsearch/index/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-elasticsearch,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/elasticsearch/index/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -33,7 +33,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-dropwizard,Dropwizard>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-dropwizard-collector,collector>> beta[]  
 |<<metricbeat-module-elasticsearch,Elasticsearch>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.2+| .2+|  |<<metricbeat-metricset-elasticsearch-node,node>> beta[]  
+.3+| .3+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
+|<<metricbeat-metricset-elasticsearch-node,node>> beta[]  
 |<<metricbeat-metricset-elasticsearch-node_stats,node_stats>> beta[]  
 |<<metricbeat-module-etcd,Etcd>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .3+| .3+|  |<<metricbeat-metricset-etcd-leader,leader>> beta[]  

metricbeat/include/list.go

@@ -36,6 +36,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/dropwizard"
 	_ "github.com/elastic/beats/metricbeat/module/dropwizard/collector"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
 	_ "github.com/elastic/beats/metricbeat/module/etcd"

metricbeat/mb/testing/data_generator.go

@@ -57,6 +57,28 @@ func WriteEvents(f mb.EventsFetcher, t testing.TB) error {
 	return nil
 }
 
+// WriteEventsReporterV2 fetches events and writes the first event to a ./_meta/data.json
+// file.
+func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB) error {
+	if !*dataFlag {
+		t.Skip("skip data generation tests")
+	}
+
+	events, errs := ReportingFetchV2(f)
+	if len(errs) > 0 {
+		return errs[0]
+	}
+
+	if len(events) == 0 {
+		return fmt.Errorf("no events were generated")
+	}
+
+	e := StandardizeEvent(f, events[0])
+
+	WriteEventToDataJSON(t, e)
+	return nil
+}
+
 // CreateFullEvent builds a full event given the data generated by a MetricSet.
 // This simulates the output of Metricbeat as if it were
 // 2016-05-23T08:05:34.853Z and the hostname is host.example.com.

metricbeat/mb/testing/modules.go

@@ -151,25 +151,38 @@ func NewReportingMetricSetV2(t testing.TB, config interface{}) mb.ReportingMetri
 	return reportingMetricSetV2
 }
 
-type capturingReporterV2 struct {
+// CapturingReporterV2 is a reporter used for testing which stores all events and errors
+type CapturingReporterV2 struct {
 	events []mb.Event
 	errs   []error
 }
 
-func (r *capturingReporterV2) Event(event mb.Event) bool {
+// Event is used to report an event
+func (r *CapturingReporterV2) Event(event mb.Event) bool {
 	r.events = append(r.events, event)
 	return true
 }
 
-func (r *capturingReporterV2) Error(err error) bool {
+// Error is used to report an error
+func (r *CapturingReporterV2) Error(err error) bool {
 	r.errs = append(r.errs, err)
 	return true
 }
 
+// GetEvents returns all reported events
+func (r *CapturingReporterV2) GetEvents() []mb.Event {
+	return r.events
+}
+
+// GetErrors returns all reported errors
+func (r *CapturingReporterV2) GetErrors() []error {
+	return r.errs
+}
+
 // ReportingFetchV2 runs the given reporting metricset and returns all of the
 // events and errors that occur during that period.
 func ReportingFetchV2(metricSet mb.ReportingMetricSetV2) ([]mb.Event, []error) {
-	r := &capturingReporterV2{}
+	r := &CapturingReporterV2{}
 	metricSet.Fetch(r)
 	return r.events, r.errs
 }

metricbeat/metricbeat.reference.yml

@@ -195,7 +195,10 @@ metricbeat.modules:
 
 #---------------------------- Elasticsearch Module ---------------------------
 - module: elasticsearch
-  metricsets: ["node", "node_stats"]
+  metricsets:
+   #- index
+   - node
+   - node_stats
   period: 10s
   hosts: ["localhost:9200"]
 

metricbeat/module/elasticsearch/_meta/config.reference.yml

@@ -1,4 +1,7 @@
 - module: elasticsearch
-  metricsets: ["node", "node_stats"]
+  metricsets:
+   #- index
+   - node
+   - node_stats
   period: 10s
   hosts: ["localhost:9200"]

metricbeat/module/elasticsearch/_meta/fields.yml

@@ -14,3 +14,8 @@
           type: keyword
           description: >
             Elasticsearch cluster name.
+
+        - name: cluster.id
+          type: keyword
+          description: >
+            Elasticsearch cluster id.