Add new metrics to CPU metricset (elastic#4969)

* metricbeat: add cpu.total.pct && cpu.total.norm.pct

Closes elastic#4898

* fix system tests

(cherry picked from commit cc790d7)

CHANGELOG.asciidoc

@@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta1...master[Check the HEAD di
 
 - Added missing mongodb configuration file to the `modules.d` folder. {pull}4870[4870]
 - Fix wrong MySQL CRUD queries timelion visualization {pull}4857[4857]
+- Add new metrics to CPU metricsset {pull}4969[4969]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -8507,6 +8507,16 @@ format: percent
 The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix.
 
 
+[float]
+=== system.cpu.total.pct
+
+type: scaled_float
+
+format: percent
+
+The percentage of CPU time spent in non-idle state.
+
+
 [float]
 === system.cpu.user.norm.pct
 
@@ -8587,6 +8597,16 @@ format: percent
 The percentage of CPU time spent in involuntary wait by the virtual CPU while the hypervisor was servicing another processor. Available only on Unix.
 
 
+[float]
+=== system.cpu.total.norm.pct
+
+type: scaled_float
+
+format: percent
+
+The percentage of CPU time spent in non-idle state.
+
+
 [float]
 === system.cpu.user.ticks
 

metricbeat/module/system/cpu/_meta/fields.yml

@@ -64,6 +64,12 @@
         was servicing another processor.
         Available only on Unix.
 
+    - name: total.pct
+      type: scaled_float
+      format: percent
+      description: >
+        The percentage of CPU time spent in non-idle state.
+
     # Normalized Percentages
     - name: user.norm.pct
       type: scaled_float
@@ -115,6 +121,13 @@
         was servicing another processor.
         Available only on Unix.
 
+    - name: total.norm.pct
+      type: scaled_float
+      format: percent
+      description: >
+        The percentage of CPU time spent in non-idle state.
+
+
     # Ticks
     - name: user.ticks
       type: long

metricbeat/module/system/cpu/cpu.go

@@ -65,6 +65,7 @@ func (m *MetricSet) Fetch() (common.MapStr, error) {
 			event.Put("nice.pct", pct.Nice)
 			event.Put("softirq.pct", pct.SoftIRQ)
 			event.Put("steal.pct", pct.Steal)
+			event.Put("total.pct", pct.Total)
 		case normalizedPercentages:
 			normalizedPct := sample.NormalizedPercentages()
 			event.Put("user.norm.pct", normalizedPct.User)
@@ -75,6 +76,7 @@ func (m *MetricSet) Fetch() (common.MapStr, error) {
 			event.Put("nice.norm.pct", normalizedPct.Nice)
 			event.Put("softirq.norm.pct", normalizedPct.SoftIRQ)
 			event.Put("steal.norm.pct", normalizedPct.Steal)
+			event.Put("total.norm.pct", normalizedPct.Total)
 		case ticks:
 			ticks := sample.Ticks()
 			event.Put("user.ticks", ticks.User)

metricbeat/module/system/util.go

@@ -47,6 +47,7 @@ type CPUPercentages struct {
 	Nice    float64
 	SoftIRQ float64
 	Steal   float64
+	Total   float64
 }
 
 type CPUTicks struct {
@@ -98,6 +99,10 @@ func cpuPercentages(s0, s1 *sigar.Cpu, numCPU int) CPUPercentages {
 		return Round(pct * float64(numCPU))
 	}
 
+	calculateTotalPct := func() float64 {
+		return Round(float64(numCPU) - calculatePct(s0.Idle, s1.Idle))
+	}
+
 	return CPUPercentages{
 		User:    calculatePct(s0.User, s1.User),
 		System:  calculatePct(s0.Sys, s1.Sys),
@@ -107,6 +112,7 @@ func cpuPercentages(s0, s1 *sigar.Cpu, numCPU int) CPUPercentages {
 		Nice:    calculatePct(s0.Nice, s1.Nice),
 		SoftIRQ: calculatePct(s0.SoftIrq, s1.SoftIrq),
 		Steal:   calculatePct(s0.Stolen, s1.Stolen),
+		Total:   calculateTotalPct(),
 	}
 }
 

metricbeat/module/system/util_test.go

@@ -52,6 +52,8 @@ func TestCPUCoresMonitorSample(t *testing.T) {
 		assert.True(t, normPct.System <= 100)
 		assert.True(t, normPct.Idle > 0)
 		assert.True(t, normPct.Idle <= 100)
+		assert.True(t, normPct.Total > 0)
+		assert.True(t, normPct.Total <= 100)
 
 		ticks := s.Ticks()
 		assert.True(t, ticks.User > 0)
@@ -110,10 +112,14 @@ func TestCPUMetricsPercentages(t *testing.T) {
 	pct := sample.NormalizedPercentages()
 	assert.EqualValues(t, .3, pct.User)
 	assert.EqualValues(t, .7, pct.System)
+	assert.EqualValues(t, .0, pct.Idle)
+	assert.EqualValues(t, 1., pct.Total)
 
 	pct = sample.Percentages()
 	assert.EqualValues(t, .3*float64(NumCPU), pct.User)
 	assert.EqualValues(t, .7*float64(NumCPU), pct.System)
+	assert.EqualValues(t, .0*float64(NumCPU), pct.Idle)
+	assert.EqualValues(t, 1.*float64(NumCPU), pct.Total)
 }
 
 func TestRound(t *testing.T) {

metricbeat/tests/system/test_processors.py

@@ -41,7 +41,7 @@ def test_drop_fields(self):
         print(cpu.keys())
         self.assertItemsEqual(self.de_dot([
             "system", "cores", "user", "softirq", "iowait",
-            "idle", "irq", "steal", "nice"
+            "idle", "irq", "steal", "nice", "total"
         ]), cpu.keys())
 
     def test_dropfields_with_condition(self):

metricbeat/tests/system/test_system.py

@@ -7,12 +7,12 @@
 import os
 
 SYSTEM_CPU_FIELDS = ["cores", "idle.pct", "iowait.pct", "irq.pct", "nice.pct",
-                     "softirq.pct", "steal.pct", "system.pct", "user.pct"]
+                     "softirq.pct", "steal.pct", "system.pct", "user.pct", "total.pct"]
 
 SYSTEM_CPU_FIELDS_ALL = ["cores", "idle.pct", "idle.ticks", "iowait.pct", "iowait.ticks", "irq.pct", "irq.ticks", "nice.pct", "nice.ticks",
                          "softirq.pct", "softirq.ticks", "steal.pct", "steal.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks",
                          "idle.norm.pct", "iowait.norm.pct", "irq.norm.pct", "nice.norm.pct", "softirq.norm.pct",
-                         "steal.norm.pct", "system.norm.pct", "user.norm.pct"]
+                         "steal.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct"]
 
 SYSTEM_LOAD_FIELDS = ["cores", "1", "5", "15", "norm.1", "norm.5", "norm.15"]