Splunk Metrics serializer

docs/DATA_FORMATS_OUTPUT.md

@@ -7,6 +7,7 @@ plugins.
 1. [InfluxDB Line Protocol](#influx)
 1. [JSON](#json)
 1. [Graphite](#graphite)
+1. [SplunkMetric](#splunkmetric)
 
 You will be able to identify the plugins with support by the presence of a
 `data_format` config option, for example, in the `file` output plugin:
@@ -209,3 +210,67 @@ reference the documentation for the specific plugin.
   ## the power of 10 less than the specified units.
   json_timestamp_units = "1s"
 ```
+
+## SplunkMetric
+
+The SplunkMetric format translates the telegraf metrics into a JSON format compatible
+with a Splunk Metrics index. Normally you would use this to send the metrics to a
+HTTP Event Collector (HEC), it follows the format specified in the document [here.](http://dev.splunk.com/view/SP-CAAAFDN#json)
+
+```toml
+[[outputs.http]]
+   ## URL is the address to send metrics to
+   url = "https://localhost:8088/services/collector"
+
+   ## Timeout for HTTP message
+   # timeout = "5s"
+
+   ## HTTP method, one of: "POST" or "PUT"
+   # method = "POST"
+
+   ## HTTP Basic Auth credentials
+   # username = "username"
+   # password = "pa$$word"
+
+   ## Optional TLS Config
+   # tls_ca = "/etc/telegraf/ca.pem"
+   # tls_cert = "/etc/telegraf/cert.pem"
+   # tls_key = "/etc/telegraf/key.pem"
+   ## Use TLS but skip chain & host verification
+   # insecure_skip_verify = false
+
+   ## Data format to output.
+   ## Each data format has it's own unique set of configuration options, read
+   ## more about them here:
+   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+   data_format = "splunkmetric"
+   # Enable hec_routing 
+   hec_routing = true
+
+   ## Additional HTTP headers
+   [outputs.http.headers]
+   # Should be set manually to "application/json" for json data_format
+      Content-Type = "application/json"
+      Authorization = "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+      X-Splunk-Request-Channel = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+```
+
+It also has the option to output a more basic JSON structure that can be used by a
+Splunk Universal Forwarder or modular input.
+
+```toml
+[[outputs.file]]
+  ## Files to write to, "stdout" is a specially handled file.
+  files = ["stdout", "/tmp/metrics.out"]
+
+  ## Data format to output.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+  data_format = "json"
+  ## hec_routing defaults to false
+  # hec_routing = false
+```
+
+Please see the [Splunk Metrics README](../plugins/serializers/splunkmetric/README.md) for additional
+information.

internal/config/config.go

@@ -1455,6 +1455,18 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error
 		}
 	}
 
+	if node, ok := tbl.Fields["hec_routing"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if b, ok := kv.Value.(*ast.Boolean); ok {
+				var err error
+				c.HecRouting, err = b.Boolean()
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+
 	delete(tbl.Fields, "influx_max_line_bytes")
 	delete(tbl.Fields, "influx_sort_fields")
 	delete(tbl.Fields, "influx_uint_support")
@@ -1463,6 +1475,7 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error
 	delete(tbl.Fields, "prefix")
 	delete(tbl.Fields, "template")
 	delete(tbl.Fields, "json_timestamp_units")
+	delete(tbl.Fields, "hec_routing")
 	return serializers.NewSerializer(c)
 }
 

plugins/serializers/registry.go

@@ -9,6 +9,7 @@ import (
 	"github.com/influxdata/telegraf/plugins/serializers/graphite"
 	"github.com/influxdata/telegraf/plugins/serializers/influx"
 	"github.com/influxdata/telegraf/plugins/serializers/json"
+	"github.com/influxdata/telegraf/plugins/serializers/splunkmetric"
 )
 
 // SerializerOutput is an interface for output plugins that are able to
@@ -60,6 +61,9 @@ type Config struct {
 
 	// Timestamp units to use for JSON formatted output
 	TimestampUnits time.Duration
+
+	// Include HEC routing fields for splunkmetric output
+	HecRouting bool
 }
 
 // NewSerializer a Serializer interface based on the given config.
@@ -73,6 +77,8 @@ func NewSerializer(config *Config) (Serializer, error) {
 		serializer, err = NewGraphiteSerializer(config.Prefix, config.Template, config.GraphiteTagSupport)
 	case "json":
 		serializer, err = NewJsonSerializer(config.TimestampUnits)
+	case "splunkmetric":
+		serializer, err = NewSplunkmetricSerializer(config.HecRouting)
 	default:
 		err = fmt.Errorf("Invalid data format: %s", config.DataFormat)
 	}
@@ -83,6 +89,10 @@ func NewJsonSerializer(timestampUnits time.Duration) (Serializer, error) {
 	return json.NewSerializer(timestampUnits)
 }
 
+func NewSplunkmetricSerializer(hec_routing bool) (Serializer, error) {
+	return splunkmetric.NewSerializer(hec_routing)
+}
+
 func NewInfluxSerializerConfig(config *Config) (Serializer, error) {
 	var sort influx.FieldSortOrder
 	if config.InfluxSortFields {

plugins/serializers/splunkmetric/README.md

@@ -0,0 +1,135 @@
+# Splunk Metrics serialzier
+
+This serializer formats and outputs the metric data in a format that can be consumed by a Splunk metrics index. It can be used to write to a file using the file output, or for sending metrics to a HEC using the standard telegraf HTTP output. If you're using the HTTP output, this serializer knows how to batch the metrics so you don't end up with an HTTP POST per metric.
+
+Th data is output in a format that conforms to the specified Splunk HEC JSON format as found here: [Send metrics in JSON format](http://dev.splunk.com/view/event-collector/SP-CAAAFDN).
+
+An example event looks like:
+```javascript
+{
+  "time": 1529708430,
+  "event": "metric",
+  "host": "patas-mbp",
+  "fields": {
+    "_value": 0.6,
+    "cpu": "cpu0",
+    "dc": "mobile",
+    "metric_name": "cpu.usage_user",
+    "user": "ronnocol"
+  }
+}
+```
+In the above snippet, the following keys are dimensions:
+* cpu
+* dc
+* user
+
+## Using with the HTTP output
+
+To send this data to a Splunk HEC, you can use the HTTP output, there are some custom headers that you need to add
+to manage the HEC authorization, here's a sample config for an HTTP output:
+
+```toml
+[[outputs.http]]
+   ## URL is the address to send metrics to
+   url = "https://localhost:8088/services/collector"
+
+   ## Timeout for HTTP message
+   # timeout = "5s"
+
+   ## HTTP method, one of: "POST" or "PUT"
+   # method = "POST"
+
+   ## HTTP Basic Auth credentials
+   # username = "username"
+   # password = "pa$$word"
+
+   ## Optional TLS Config
+   # tls_ca = "/etc/telegraf/ca.pem"
+   # tls_cert = "/etc/telegraf/cert.pem"
+   # tls_key = "/etc/telegraf/key.pem"
+   ## Use TLS but skip chain & host verification
+   # insecure_skip_verify = false
+
+   ## Data format to output.
+   ## Each data format has it's own unique set of configuration options, read
+   ## more about them here:
+   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+   data_format = "splunkmetric"
+    ## Provides time, index, source overrides for the HEC
+   hec_routing = true
+
+   ## Additional HTTP headers
+    [outputs.http.headers]
+   # Should be set manually to "application/json" for json data_format
+      Content-Type = "application/json"
+      Authorization = "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+      X-Splunk-Request-Channel = "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+```
+
+## Overrides
+You can override the default values for the HEC token you are using by adding additional tags to the config file.
+
+The following aspects of the token can be overriden with tags:
+* index
+* source
+
+You can either use `[global_tags]` or using a more advanced configuration as documented [here](https://github.com/influxdata/telegraf/blob/master/docs/CONFIGURATION.md).
+
+Such as this example which overrides the index just on the cpu metric:
+```toml
+[[inputs.cpu]]
+  percpu = false
+  totalcpu = true
+  [inputs.cpu.tags]
+    index = "cpu_metrics"
+```
+
+## Using with the File output
+
+You can use the file output when running telegraf on a machine with a Splunk forwarder.
+
+A sample event when `hec_routing` is false (or unset) looks like:
+```javascript
+{
+    "_value": 0.6,
+    "cpu": "cpu0",
+    "dc": "mobile",
+    "metric_name": "cpu.usage_user",
+    "user": "ronnocol",
+    "time": 1529708430
+}
+```
+Data formatted in this manner can be ingested with a simple `props.conf` file that
+looks like this:
+
+```ini
+[telegraf]
+category = Metrics
+description = Telegraf Metrics
+pulldown_type = 1
+DATETIME_CONFIG =
+NO_BINARY_CHECK = true
+SHOULD_LINEMERGE = true
+disabled = false
+INDEXED_EXTRACTIONS = json
+KV_MODE = none
+TIMESTAMP_FIELDS = time
+TIME_FORMAT = %s.%3N
+```
+
+An example configuration of a file based output is: 
+
+```toml
+ # Send telegraf metrics to file(s)
+[[outputs.file]]
+   ## Files to write to, "stdout" is a specially handled file.
+   files = ["/tmp/metrics.out"]
+
+   ## Data format to output.
+   ## Each data format has its own unique set of configuration options, read
+   ## more about them here:
+   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+   data_format = "splunkmetric"
+   hec_routing = false
+```