Log dropped events at the debug level (#9251)

Log events dropped by encoding problems at the debug level.

CHANGELOG.asciidoc

@@ -35,6 +35,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 *Affecting all Beats*
 
 - Fix autodiscover configurations stopping when metadata is missing. {pull}8851[8851]
+- Log events at the debug level when dropped by encoding problems. {pull}9251[9251]
 
 *Auditbeat*
 

libbeat/outputs/console/console.go

@@ -129,6 +129,7 @@ func (c *console) publishEvent(event *publisher.Event) bool {
 		}
 
 		logp.Critical("Unable to encode event: %v", err)
+		logp.Debug("console", "Failed event: %v", event)
 		return false
 	}
 

libbeat/outputs/elasticsearch/client.go

@@ -380,6 +380,7 @@ func bulkEncodePublishRequest(
 		}
 		if err := body.Add(meta, event); err != nil {
 			logp.Err("Failed to encode event: %s", err)
+			logp.Debug("elasticsearch", "Failed event: %v", event)
 			continue
 		}
 		okEvents = append(okEvents, data[i])

libbeat/outputs/fileout/file.go

@@ -126,6 +126,7 @@ func (out *fileOutput) Publish(
 			} else {
 				logp.Warn("Failed to serialize the event: %v", err)
 			}
+			logp.Debug("file", "Failed event: %v", event)
 
 			dropped++
 			continue

libbeat/outputs/kafka/client.go

@@ -179,6 +179,7 @@ func (c *client) getEventMessage(data *publisher.Event) (*message, error) {
 
 	serializedEvent, err := c.codec.Encode(c.index, event)
 	if err != nil {
+		logp.Debug("kafka", "Failed event: %v", event)
 		return nil, err
 	}
 

libbeat/outputs/logstash/enc.go

@@ -24,7 +24,11 @@ import (
 
 func makeLogstashEventEncoder(info beat.Info, escapeHTML bool, index string) func(interface{}) ([]byte, error) {
 	enc := json.New(false, escapeHTML, info.Version)
-	return func(event interface{}) ([]byte, error) {
-		return enc.Encode(index, event.(*beat.Event))
+	return func(event interface{}) (d []byte, err error) {
+		d, err = enc.Encode(index, event.(*beat.Event))
+		if err != nil {
+			debugf("Failed to encode event: %v", event)
+		}
+		return
 	}
 }

libbeat/outputs/redis/client.go

@@ -312,6 +312,7 @@ func serializeEvents(
 		serializedEvent, err := codec.Encode(index, &d.Content)
 		if err != nil {
 			logp.Err("Encoding event failed with error: %v", err)
+			logp.Debug("redis", "Failed event: %v", d.Content)
 			goto failLoop
 		}
 
@@ -329,6 +330,7 @@ failLoop:
 		serializedEvent, err := codec.Encode(index, &d.Content)
 		if err != nil {
 			logp.Err("Encoding event failed with error: %v", err)
+			logp.Debug("redis", "Failed event: %v", d.Content)
 			i++
 			continue
 		}