Force ECS and JSON logging for libbeat/logp

CHANGELOG-developer.next.asciidoc

@@ -56,6 +56,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 - Removed the `common.Float` type. {issue}28279[28279] {pull}28280[28280] {pull}28376[28376]
+- libbeat.logp package forces ECS compliant logs. Logs are also JSON formatted by default. {issue}15544[15544] {pull}28573[28573]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve stats API {pull}27963[27963]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
 - Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
+- Libbeat: logp package forces ECS compliant logs. Logs are also JSON formatted by default. {issue}15544[15544] {pull}28573[28573]
 - Previously, RE2 and thus Golang had a bug where `(|a)*` matched more characters than `(|a)+`. To stay consistent with PCRE, the bug was fixed. Configurations that rely on the old, buggy behaviour has to be adjusted. See more about Golang bug: https://github.com/golang/go/issues/46123 {pull}27543[27543]
 
 *Auditbeat*

auditbeat/auditbeat.reference.yml

@@ -1451,11 +1451,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

auditbeat/tests/system/test_file_integrity.py

@@ -98,7 +98,7 @@ def test_non_recursive(self):
 
             # wait until file1 is reported before deleting. Otherwise the hash
             # might not be calculated
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file1)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file1)), ignore_case=True)
 
             os.unlink(file1)
 
@@ -107,8 +107,9 @@ def test_non_recursive(self):
             file3 = os.path.join(subdir, "other_file.txt")
             self.create_file(file3, "not reported.")
 
-            self.wait_log_contains("\"deleted\"")
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(subdir)), ignore_case=True)
+            # log entries are JSON formatted, this value shows up as an escaped json string.
+            self.wait_log_contains("\\\"deleted\\\"")
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(subdir)), ignore_case=True)
             self.wait_output(3)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir.lower() for obj in self.read_output()))
@@ -157,7 +158,7 @@ def test_recursive(self):
             # wait until the directories to watch are printed in the logs
             # this happens when the file_integrity module starts
             self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True)
-            self.wait_log_contains("\"recursive\": true")
+            self.wait_log_contains("\"recursive\":true")
 
             # auditbeat_test/subdir/
             subdir = os.path.join(dirs[0], "subdir")
@@ -173,7 +174,7 @@ def test_recursive(self):
             file2 = os.path.join(subdir2, "more.txt")
             self.create_file(file2, "")
 
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file2)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file2)), ignore_case=True)
             self.wait_output(4)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir2.lower() for obj in self.read_output()))

filebeat/filebeat.reference.yml

@@ -2363,11 +2363,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

heartbeat/heartbeat.reference.yml

@@ -1597,11 +1597,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

journalbeat/journalbeat.reference.yml

@@ -1394,11 +1394,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Journalbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -75,8 +75,3 @@ logging.files:
 
 # Set to true to log messages in JSON format.
 #logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false

libbeat/logp/config.go

@@ -26,11 +26,10 @@ import (
 // Config contains the configuration options for the logger. To create a Config
 // from a common.Config use logp/config.Build.
 type Config struct {
-	Beat       string   `config:",ignore"`        // Name of the Beat (for default file name).
-	JSON       bool     `config:"json"`           // Write logs as JSON.
-	Level      Level    `config:"level"`          // Logging level (error, warning, info, debug).
-	Selectors  []string `config:"selectors"`      // Selectors for debug level logging.
-	ECSEnabled bool     `config:"ecs" yaml:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line
+	Beat      string   `config:",ignore"`   // Name of the Beat (for default file name).
+	JSON      bool     `config:"json"`      // Write logs as JSON.
+	Level     Level    `config:"level"`     // Logging level (error, warning, info, debug).
+	Selectors []string `config:"selectors"` // Selectors for debug level logging.
 
 	toObserver  bool
 	toIODiscard bool
@@ -77,6 +76,7 @@ const (
 func DefaultConfig(environment Environment) Config {
 	return Config{
 		Level: defaultLevel,
+		JSON:  true,
 		Files: FileConfig{
 			Suffix:          file.SuffixCount,
 			MaxSize:         10 * 1024 * 1024,

libbeat/logp/core.go

@@ -196,7 +196,7 @@ func makeOptions(cfg Config) []zap.Option {
 	if cfg.development {
 		options = append(options, zap.Development())
 	}
-	if cfg.ECSEnabled {
+	if cfg.Beat != "" {
 		fields := []zap.Field{
 			zap.String("service.name", cfg.Beat),
 		}
@@ -254,10 +254,7 @@ func newCore(cfg Config, enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapco
 	return wrappedCore(cfg, zapcore.NewCore(enc, ws, enab))
 }
 func wrappedCore(cfg Config, core zapcore.Core) zapcore.Core {
-	if cfg.ECSEnabled {
-		return ecszap.WrapCore(core)
-	}
-	return core
+	return ecszap.WrapCore(core)
 }
 
 func globalLogger() *zap.Logger {

libbeat/logp/core_test.go

@@ -152,7 +152,6 @@ func TestLoggingECSFields(t *testing.T) {
 		Beat:        "beat1",
 		Level:       DebugLevel,
 		development: true,
-		ECSEnabled:  true,
 		Files: FileConfig{
 			Name: "beat1.log",
 		},

libbeat/logp/encoding.go

@@ -54,9 +54,7 @@ func buildEncoder(cfg Config) zapcore.Encoder {
 		encCreator = zapcore.NewConsoleEncoder
 	}
 
-	if cfg.ECSEnabled {
-		encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
-	}
+	encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
 	return encCreator(encCfg)
 }
 

libbeat/tests/system/test_logging.py

@@ -25,26 +25,13 @@ def assert_contains_ecs_log(self, logfile=None):
         assert self.log_contains(ecs_message_log, logfile=logfile)
         assert self.log_contains(ecs_log_level_log, logfile=logfile)
 
-    def assert_not_contains_ecs_log(self, logfile=None):
-        assert not self.log_contains(ecs_version_log, logfile=logfile)
-        assert not self.log_contains(ecs_timestamp_log, logfile=logfile)
-        assert not self.log_contains(ecs_log_level_log, logfile=logfile)
-
-    def test_console_default(self):
-        """
-        logs to console with default format
-        """
-        self.run_beat_with_args("mockbeat start running", logging_args=["-e"])
-        self.assert_not_contains_ecs_log()
-
     def test_console_ecs(self):
         """
         logs to console with ECS format
         """
         self.run_beat_with_args("mockbeat start running",
                                 logging_args=["-e"],
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=true"])
+                                extra_args=["-E", "logging.json=true"])
         self.assert_contains_ecs_log()
 
     def test_file_default(self):
@@ -53,15 +40,13 @@ def test_file_default(self):
         """
         self.run_beat_with_args("Mockbeat is alive!",
                                 logging_args=[],
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=false"])
-        self.assert_not_contains_ecs_log(logfile="logs/mockbeat")
+                                extra_args=["-E", "logging.json=true"])
+        self.assert_contains_ecs_log(logfile="logs/mockbeat")
 
     def test_file_ecs(self):
         """
         logs to file with ECS format
         """
         self.run_beat_with_args("Mockbeat is alive!",
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=true"])
+                                extra_args=["-E", "logging.json=true"])
         self.assert_contains_ecs_log(logfile="logs/mockbeat")

libbeat/tests/system/test_template.py

@@ -107,7 +107,7 @@ def test_json_template(self):
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
         self.wait_until(lambda: self.log_contains("Loading json template from file"))
-        self.wait_until(lambda: self.log_contains('Template with name "bla" loaded.'))
+        self.wait_until(lambda: self.log_contains('Template with name \\\"bla\\\" loaded.'))
         proc.check_kill_and_wait()
 
         result = es.transport.perform_request('GET', '/_template/' + template_name)
@@ -149,7 +149,7 @@ def test_template_default(self):
         self.render_config()
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
-        self.wait_until(lambda: self.log_contains('Template with name "mockbeat-9.9.9" loaded.'))
+        self.wait_until(lambda: self.log_contains('Template with name \\\"mockbeat-9.9.9\\\" loaded.'))
         self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
         proc.check_kill_and_wait()
 

metricbeat/metricbeat.reference.yml

@@ -2266,11 +2266,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Metricbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

metricbeat/module/elasticsearch/test_elasticsearch.py

@@ -135,7 +135,7 @@ def test_xpack_cluster_stats(self):
             }
         }])
         proc = self.start_beat()
-        self.wait_log_contains('"dataset": "elasticsearch.cluster.stats"')
+        self.wait_log_contains('\\"dataset\\": \\"elasticsearch.cluster.stats\\"')
 
         proc.check_kill_and_wait()
         self.assert_no_logged_warnings()

packetbeat/packetbeat.reference.yml

@@ -1946,11 +1946,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Packetbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

winlogbeat/winlogbeat.reference.yml

@@ -1374,11 +1374,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Winlogbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/auditbeat/auditbeat.reference.yml

@@ -1507,11 +1507,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl

@@ -197,9 +197,4 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 {{template "providers.yml.tmpl" .}}

x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl

@@ -197,9 +197,4 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 {{template "providers.yml.tmpl" .}}

x-pack/elastic-agent/_meta/elastic-agent.yml

@@ -191,8 +191,3 @@ agent.logging.to_stderr: true
 
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false

x-pack/elastic-agent/elastic-agent.reference.yml

@@ -203,11 +203,6 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 # Providers
 
 # Providers supply the key/values pairs that are used for variable substitution

x-pack/elastic-agent/elastic-agent.yml

@@ -203,11 +203,6 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 # Providers
 
 # Providers supply the key/values pairs that are used for variable substitution

x-pack/filebeat/filebeat.reference.yml

@@ -4504,11 +4504,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/functionbeat/functionbeat.reference.yml

@@ -1245,11 +1245,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Functionbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/heartbeat/heartbeat.reference.yml

@@ -1597,11 +1597,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/metricbeat/metricbeat.reference.yml

@@ -2796,11 +2796,6 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Metricbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The