Check fields type and format

CHANGELOG-developer.next.asciidoc

@@ -47,4 +47,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Introduce beat.OutputChooses publisher mode. {pull}12996[12996]
 - Ensure that beat.Processor, beat.ProcessorList, and processors.ProcessorList are compatible and can be composed more easily. {pull}12996[12996]
 - Add support to close beat.Client via beat.CloseRef (a subset of context.Context). {pull}13031[13031]
+- Add checks for types and formats used in fields definitions in `fields.yml` files. {pull}13188[13188]
 - Makefile included in generator copies files from beats repository using `git archive` instead of cp. {pull}13193[13193]

CHANGELOG.next.asciidoc

@@ -86,6 +86,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix seccomp policy preventing some features to function properly on 32bit Linux systems. {issue}12990[12990] {pull}13008[13008]
 - Fix unexpected stops on docker autodiscover when a container is restarted before `cleanup_timeout`. {issue}12962[12962] {pull}13127[13127]
 - Fix install-service.ps1's ability to set Windows service's delay start configuration. {pull}13173[13173]
+- Fix some incorrect types and formats in field.yml files. {pull}13188[13188]
 - Load DLLs only from Windows system directory. {pull}13234[13234]
 
 *Auditbeat*

libbeat/kibana/testdata/extensive/fields.yml

@@ -1110,7 +1110,7 @@
           fields:
             - name: kernel.pct
               type: scaled_float
-              format: percentage
+              format: percent
               description: >
                 The system kernel consumed by the Docker server.
             - name: kernel.ticks
@@ -1119,23 +1119,23 @@
                 CPU kernel ticks.
             - name: system.pct
               type: scaled_float
-              format: percentage
+              format: percent
               description: >
             - name: system.ticks
               type: long
               description: >
                 CPU system ticks.
             - name: user.pct
               type: scaled_float
-              format: percentage
+              format: percent
               description: >
             - name: user.ticks
               type: long
               description: >
                 CPU user ticks
             - name: total.pct
               type: scaled_float
-              format: percentage
+              format: percent
               description: >
                 Total CPU usage.
             # TODO: how to document cpu list?
@@ -1309,7 +1309,7 @@
                     Total memory resident set size.
                 - name: pct
                   type: scaled_float
-                  format: percentage
+                  format: percent
                   description: >
                     Memory resident set size percentage.
             - name: usage
@@ -1324,7 +1324,7 @@
                     Max memory usage.
                 - name: pct
                   type: scaled_float
-                  format: percentage
+                  format: percent
                   description: >
                     Memory usage percentage.
                 - name: total
@@ -2061,7 +2061,7 @@
 
             - name: throttle.pct
               type: scaled_float
-              format: percentage
+              format: percent
               description: >
                 Current throttle percentage for the server when slowstart
                 is active, or no value if slowstart is inactive.
@@ -4885,7 +4885,7 @@
                 Number of consumers.
             - name: consumers.utilisation.pct
               type: long
-              format: percentage
+              format: percent
               description: >
                 Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count.
             - name: messages.total.count
@@ -5000,8 +5000,8 @@
               fields:
                 - name: used.value
                   type: long
-                  description: >
                   format: bytes
+                  description: >
                     Used memory.
 
                 - name: used.rss

libbeat/mapping/field.go

@@ -110,15 +110,71 @@ func (d *DynamicType) Unpack(s string) error {
 
 // Validate ensures objectTypeParams are not mixed with top level objectType configuration
 func (f *Field) Validate() error {
-	if len(f.ObjectTypeParams) == 0 {
-		return nil
+	if err := f.validateType(); err != nil {
+		return errors.Wrapf(err, "incorrect type configuration for field '%s'", f.Name)
+	}
+	if len(f.ObjectTypeParams) > 0 {
+		if f.ScalingFactor != 0 || f.ObjectTypeMappingType != "" || f.ObjectType != "" {
+			return errors.New("mixing top level objectType configuration with array of object type configurations is forbidden")
+		}
 	}
-	if f.ScalingFactor != 0 || f.ObjectTypeMappingType != "" || f.ObjectType != "" {
-		return errors.New("mixing top level objectType configuration with array of object type configurations is forbidden")
+	return nil
+}
+
+func (f *Field) validateType() error {
+	switch strings.ToLower(f.Type) {
+	case "text", "keyword":
+		return stringType.validate(f.Format)
+	case "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float":
+		return numberType.validate(f.Format)
+	case "date", "date_nanos":
+		return dateType.validate(f.Format)
+	case "geo_point":
+		return geoPointType.validate(f.Format)
+	case "boolean", "binary", "ip", "alias", "array":
+		if f.Format != "" {
+			return fmt.Errorf("no format expected for field %s, found: %s", f.Name, f.Format)
+		}
+	case "object", "group", "nested":
+		// No check for them yet
+	case "":
+		// Module keys, not used as fields
+	default:
+		// There are more types, not being used by beats, to be added if needed
+		return fmt.Errorf("unexpected type '%s' for field '%s'", f.Type, f.Name)
 	}
 	return nil
 }
 
+type fieldTypeGroup struct {
+	name string
+
+	// formatters used in Kibana, taken from https://www.elastic.co/guide/en/kibana/7.3/managing-fields.html
+	// Value shown in Kibana docs and UI is not always the same as the
+	// internal value, e.g. `percent` appears as `Percentage` in docs
+	// and UI. We have to use here the internal value.
+	formatters []string
+}
+
+var (
+	stringType   = fieldTypeGroup{"string", []string{"string", "url"}}
+	numberType   = fieldTypeGroup{"number", []string{"string", "url", "bytes", "duration", "number", "percent", "color"}}
+	dateType     = fieldTypeGroup{"date", []string{"string", "url", "date"}}
+	geoPointType = fieldTypeGroup{"geo_point", []string{"geo_point"}}
+)
+
+func (g *fieldTypeGroup) validate(formatter string) error {
+	if formatter == "" {
+		return nil
+	}
+	for _, expected := range g.formatters {
+		if expected == formatter {
+			return nil
+		}
+	}
+	return fmt.Errorf("unexpected formatter for %s type, expected one of: %s", g.name, strings.Join(g.formatters, ", "))
+}
+
 func LoadFieldsYaml(path string) (Fields, error) {
 	var keys []Field
 

libbeat/scripts/cmd/global_fields/main.go

@@ -24,6 +24,7 @@ import (
 	"path/filepath"
 
 	"github.com/elastic/beats/libbeat/generator/fields"
+	"github.com/elastic/beats/libbeat/mapping"
 )
 
 func main() {
@@ -95,6 +96,12 @@ func main() {
 		os.Exit(3)
 	}
 
+	_, err = mapping.LoadFieldsYaml(output)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Generated global fields.yml file for %s is invalid: %+v\n", name, err)
+		os.Exit(3)
+	}
+
 	outputPath, _ := filepath.Abs(output)
 	if err != nil {
 		outputPath = output

metricbeat/docs/fields.asciidoc

@@ -4798,7 +4798,7 @@ Percentage of time in kernel space.
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -4820,7 +4820,7 @@ Percentage of total CPU time in the system.
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -4842,7 +4842,7 @@ Percentage of time in user space.
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -4864,7 +4864,7 @@ Total CPU usage.
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -4876,7 +4876,7 @@ Percentage of CPU time in this core.
 
 type: object
 
-format: percentage
+format: percent
 
 --
 
@@ -5486,7 +5486,7 @@ Memory resident set size percentage.
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -5517,7 +5517,7 @@ Memory usage percentage.
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -12010,7 +12010,7 @@ Current throttle percentage for the server when slowstart is active, or no value
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -14136,7 +14136,7 @@ CPU usage as a percentage of the total node allocatable CPU
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -14148,7 +14148,7 @@ CPU usage as a percentage of the defined limit for the container (or total node
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -14264,7 +14264,7 @@ Memory usage as a percentage of the total node allocatable memory
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -14276,7 +14276,7 @@ Memory usage as a percentage of the defined limit for the container (or total no
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -15253,7 +15253,7 @@ CPU usage as a percentage of the total node CPU
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -15265,7 +15265,7 @@ CPU usage as a percentage of the defined limit for the pod containers (or total
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -15291,7 +15291,7 @@ Memory usage as a percentage of the total node allocatable memory
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -15303,7 +15303,7 @@ Memory usage as a percentage of the defined limit for the pod containers (or tot
 
 type: scaled_float
 
-format: percentage
+format: percent
 
 --
 
@@ -22952,8 +22952,6 @@ The date and time FPM has started.
 
 type: date
 
-format: epoch_second
-
 --
 
 [float]
@@ -22993,8 +22991,6 @@ The date and time the process has started
 
 type: date
 
-format: epoch_second
-
 --
 
 *`php_fpm.process.start_since`*::
@@ -24683,7 +24679,7 @@ Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately
 
 type: long
 
-format: percentage
+format: percent
 
 --
 
@@ -24957,11 +24953,12 @@ Redis memory stats.
 *`redis.info.memory.used.value`*::
 +
 --
+Total number of bytes allocated by Redis.
 
 
 type: long
 
-format: bytes Total number of bytes allocated by Redis.
+format: bytes
 
 --