Log: Add experimental option "redirect_stderr" (elastic#8430)

This patch adds a new option to log file output: `redirect_stderr`.

When it is set, any messages written to stderr from the beat process
will be written to the log file. This allows to capture panics that are
not catched by logp.Recover.

CHANGELOG.asciidoc

@@ -95,6 +95,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 - Allow Bus to buffer events in case listeners are not configured. {pull}8527[8527]
 - Dissect will now flag event on parsing error. {pull}8751[8751]
 - add_cloud_metadata initialization is performed asynchronously to avoid delays on startup. {pull}8845[8845]
+- Added the `redirect_stderr` option that allows panics to be logged to log files. {pull}8430[8430]
 
 *Auditbeat*
 

libbeat/common/file/rotator.go

@@ -68,6 +68,7 @@ type Rotator struct {
 	log             Logger // Optional Logger (may be nil).
 	interval        time.Duration
 	intervalRotator *intervalRotator // Optional, may be nil
+	redirectStderr  bool
 
 	file  *os.File
 	size  uint
@@ -123,6 +124,14 @@ func Interval(d time.Duration) RotatorOption {
 	}
 }
 
+// RedirectStderr causes all writes to standard error to be redirected
+// to this rotator.
+func RedirectStderr(redirect bool) RotatorOption {
+	return func(r *Rotator) {
+		r.redirectStderr = redirect
+	}
+}
+
 // NewFileRotator returns a new Rotator.
 func NewFileRotator(filename string, options ...RotatorOption) (*Rotator, error) {
 	r := &Rotator{
@@ -277,7 +286,9 @@ func (r *Rotator) openFile() error {
 	if err != nil {
 		return errors.Wrap(err, "failed to open new file")
 	}
-
+	if r.redirectStderr {
+		RedirectStandardError(r.file)
+	}
 	return nil
 }
 

libbeat/common/file/stderr_other.go

@@ -0,0 +1,31 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build !windows
+
+package file
+
+import (
+	"os"
+	"syscall"
+)
+
+// RedirectStandardError causes all standard error output to be directed to the
+// given file.
+func RedirectStandardError(toFile *os.File) error {
+	return syscall.Dup2(int(toFile.Fd()), 2)
+}

libbeat/common/file/stderr_windows.go

@@ -0,0 +1,30 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package file
+
+import (
+	"os"
+
+	"golang.org/x/sys/windows"
+)
+
+// RedirectStandardError causes all standard error output to be directed to the
+// given file.
+func RedirectStandardError(toFile *os.File) error {
+	return windows.SetStdHandle(windows.STD_ERROR_HANDLE, windows.Handle(toFile.Fd()))
+}

libbeat/docs/loggingconfig.asciidoc

@@ -169,6 +169,16 @@ unix epoch. Defaults to disabled.
 
 When true, logs messages in JSON format. The default is false.
 
+[float]
+==== `logging.files.redirect_stderr` experimental[]
+
+When true, diagnostic messages printed to {beatname_uc}'s standard error output
+will also be logged to the log file. This can be helpful in situations were
+{beatname_uc} terminates unexpectedly because an error has been detected by
+Go's runtime but diagnostic information is not present in the log file.
+This feature is only available when logging to files (`logging.to_files` is true).
+Disabled by default.
+
 [float]
 === Logging format
 

libbeat/logp/config.go

@@ -42,12 +42,13 @@ type Config struct {
 
 // FileConfig contains the configuration options for the file output.
 type FileConfig struct {
-	Path        string        `config:"path"`
-	Name        string        `config:"name"`
-	MaxSize     uint          `config:"rotateeverybytes" validate:"min=1"`
-	MaxBackups  uint          `config:"keepfiles" validate:"max=1024"`
-	Permissions uint32        `config:"permissions"`
-	Interval    time.Duration `config:"interval"`
+	Path           string        `config:"path"`
+	Name           string        `config:"name"`
+	MaxSize        uint          `config:"rotateeverybytes" validate:"min=1"`
+	MaxBackups     uint          `config:"keepfiles" validate:"max=1024"`
+	Permissions    uint32        `config:"permissions"`
+	Interval       time.Duration `config:"interval"`
+	RedirectStderr bool          `config:"redirect_stderr"`
 }
 
 var defaultConfig = Config{

libbeat/logp/core.go

@@ -197,6 +197,7 @@ func makeFileOutput(cfg Config) (zapcore.Core, error) {
 		file.MaxBackups(cfg.Files.MaxBackups),
 		file.Permissions(os.FileMode(cfg.Files.Permissions)),
 		file.Interval(cfg.Files.Interval),
+		file.RedirectStderr(cfg.Files.RedirectStderr),
 	)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create file rotator")