Rename source field in Filebeat

CHANGELOG.asciidoc

@@ -21,11 +21,12 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Rename beat.name to agent.type, beat.hostname to agent.hostname, beat.version to agent.version.
 
 *Filebeat*
-- Rename `fileset.name` to `event.name`.
-- Rename `fileset.module` to `event.module`.
 
+- Rename `fileset.name` to `event.name`. {pull}8879[8879]
+- Rename `fileset.module` to `event.module`. {pull}8879[8879]
+- Rename source to log.file.path and log.source.ip {pull}8902[8902]
 - Remove the deprecated `prospector(s)` option in the configuration use `input(s)` instead. {pull}8909[8909]
-- Rename `offset` to `log.offset`.
+- Rename `offset` to `log.offset`. {pull}8923[8923]
 
 *Heartbeat*
 

dev-tools/ecs-migration.yml

@@ -24,6 +24,10 @@
 - from: fileset.module
   to: event.module
   alias: true
+
+- from: source
+  to: ["log.file.path", "log.source.ip"]
+  alias: false
   copy_to: false
 
 - from: beat.name

filebeat/_meta/fields.common.yml

@@ -3,13 +3,20 @@
   description: >
     Contains log file lines.
   fields:
-    - name: source
+
+    - name: log.file.path
       type: keyword
-      required: true
+      required: false
       description: >
         The file from which the line was read. This field contains the absolute path to the file.
         For example: `/var/log/system.log`.
 
+    - name: log.source.ip
+      type: ip
+      required: false
+      description: >
+        Source IP from which the log event was read / sent from.
+
     - name: log.offset
       type: long
       required: false

filebeat/docs/fields.asciidoc

@@ -4609,16 +4609,28 @@ Contains log file lines.
 
 
 
-*`source`*::
+*`log.file.path`*::
 +
 --
 type: keyword
 
-required: True
+required: False
 
 The file from which the line was read. This field contains the absolute path to the file. For example: `/var/log/system.log`.
 
 
+--
+
+*`log.source.ip`*::
++
+--
+type: ip
+
+required: False
+
+Source IP from which the log event was read / sent from.
+
+
 --
 
 *`log.offset`*::

filebeat/input/log/harvester.go

@@ -306,9 +306,11 @@ func (h *Harvester) Run() error {
 		// Check if data should be added to event. Only export non empty events.
 		if !message.IsEmpty() && h.shouldExportLine(text) {
 			fields := common.MapStr{
-				"source": state.Source,
 				"log": common.MapStr{
 					"offset": startingOffset, // Offset here is the offset before the starting char.
+					"file": common.MapStr{
+						"path": state.Source,
+					},
 				},
 			}
 			fields.DeepUpdate(message.Fields)

filebeat/input/syslog/input.go

@@ -203,7 +203,11 @@ func (p *Input) Wait() {
 func createEvent(ev *event, metadata inputsource.NetworkMetadata, timezone *time.Location, log *logp.Logger) *beat.Event {
 	f := common.MapStr{
 		"message": strings.TrimRight(ev.Message(), "\n"),
-		"source":  metadata.RemoteAddr.String(),
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"ip": metadata.RemoteAddr.String(),
+			},
+		},
 	}
 
 	syslog := common.MapStr{}

filebeat/input/syslog/input_test.go

@@ -40,7 +40,11 @@ func TestWhenPriorityIsSet(t *testing.T) {
 	event := createEvent(e, m, time.Local, logp.NewLogger("syslog"))
 
 	expected := common.MapStr{
-		"source":   "127.0.0.1",
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"ip": "127.0.0.1",
+			},
+		},
 		"message":  "hello world",
 		"hostname": "wopr",
 		"process": common.MapStr{
@@ -69,7 +73,11 @@ func TestWhenPriorityIsNotSet(t *testing.T) {
 	m := dummyMetadata()
 	event := createEvent(e, m, time.Local, logp.NewLogger("syslog"))
 	expected := common.MapStr{
-		"source":   "127.0.0.1",
+		"log": common.MapStr{
+			"source": common.MapStr{
+				"ip": "127.0.0.1",
+			},
+		},
 		"message":  "hello world",
 		"hostname": "wopr",
 		"process": common.MapStr{

filebeat/input/tcp/input.go

@@ -131,7 +131,11 @@ func createEvent(raw []byte, metadata inputsource.NetworkMetadata) *util.Data {
 		Timestamp: time.Now(),
 		Fields: common.MapStr{
 			"message": string(raw),
-			"source":  metadata.RemoteAddr.String(),
+			"log": common.MapStr{
+				"source": common.MapStr{
+					"ip": metadata.RemoteAddr.String(),
+				},
+			},
 		},
 	}
 	return data

filebeat/input/tcp/input_test.go

@@ -42,6 +42,6 @@ func TestCreateEvent(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, string(message), m)
 
-	from, _ := event.GetValue("source")
+	from, _ := event.GetValue("log.source.ip")
 	assert.Equal(t, ip, from)
 }

filebeat/input/udp/input.go

@@ -74,7 +74,11 @@ func NewInput(
 			},
 			Fields: common.MapStr{
 				"message": string(data),
-				"source":  metadata.RemoteAddr.String(),
+				"log": common.MapStr{
+					"source": common.MapStr{
+						"ip": metadata.RemoteAddr.String(),
+					},
+				},
 			},
 		}
 		forwarder.Send(e)

filebeat/module/icinga/startup/test/test.log-expected.json

@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-11-06T15:26:20.085Z", 
+        "@timestamp": "2018-11-06T18:29:13.579Z", 
         "event.dataset": "startup", 
         "event.module": "icinga", 
         "icinga.startup.facility": "cli", 
@@ -10,7 +10,7 @@
         "log.offset": 0
     }, 
     {
-        "@timestamp": "2018-11-06T15:26:20.085Z", 
+        "@timestamp": "2018-11-06T18:29:13.579Z", 
         "event.dataset": "startup", 
         "event.module": "icinga", 
         "icinga.startup.facility": "cli", 

filebeat/tests/system/test_modules.py

@@ -194,7 +194,7 @@ def clean_keys(obj):
     # The create timestamps area always new
     time_keys = ["read_timestamp", "event.created"]
     # source path and beat.version can be different for each run
-    other_keys = ["source", "agent.version"]
+    other_keys = ["log.file.path", "agent.version"]
 
     for key in host_keys + time_keys + other_keys:
         delete_key(obj, key)

filebeat/tests/system/test_processors.py

@@ -66,7 +66,7 @@ def test_drop_event(self):
             path=os.path.abspath(self.working_dir) + "/test*.log",
             processors=[{
                 "drop_event": {
-                    "when": "contains.source: test1",
+                    "when": "contains.log.file.path: test1",
                 },
             }]
         )

filebeat/tests/system/test_syslog.py

@@ -100,4 +100,4 @@ def assert_syslog(self, syslog):
         assert syslog["syslog.priority"] == 13
         assert syslog["syslog.severity_label"] == "Notice"
         assert syslog["syslog.facility_label"] == "user-level"
-        assert len(syslog["source"]) > 0
+        assert len(syslog["log.source.ip"]) > 0