Configuration to allow users to retry loading dashboards (#6560)

Configuration to allow users to retry loading dashboards if Kibana is not reachable, instead of exiting right away.

CHANGELOG.asciidoc

@@ -196,6 +196,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Experimental feature setup.template.append_fields added. {pull}6024[6024]
 - Add appender support to autodiscover {pull}6469[6469]
 - Add add_host_metadata processor {pull}5968[5968]
+- Retry configuration to load dashboards if Kibana is not reachable when the beat starts. {pull}6560[6560]
 
 *Auditbeat*
 

auditbeat/auditbeat.reference.yml

@@ -744,6 +744,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

filebeat/filebeat.reference.yml

@@ -1223,6 +1223,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

heartbeat/heartbeat.reference.yml

@@ -853,6 +853,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

libbeat/_meta/config.reference.yml

@@ -639,6 +639,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

libbeat/cmd/instance/beat.go

@@ -1,6 +1,7 @@
 package instance
 
 import (
+	"context"
 	cryptRand "crypto/rand"
 	"encoding/json"
 	"flag"
@@ -294,9 +295,10 @@ func (b *Beat) launch(bt beat.Creator) error {
 		return beat.GracefulExit
 	}
 
-	svc.HandleSignals(beater.Stop)
+	ctx, cancel := context.WithCancel(context.Background())
+	svc.HandleSignals(beater.Stop, cancel)
 
-	err = b.loadDashboards(false)
+	err = b.loadDashboards(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -382,7 +384,8 @@ func (b *Beat) Setup(bt beat.Creator, template, dashboards, machineLearning bool
 		}
 
 		if dashboards {
-			err = b.loadDashboards(true)
+			fmt.Println("Loading dashboards (Kibana must be running and reachable)")
+			err = b.loadDashboards(context.Background(), true)
 			if err != nil {
 				return err
 			}
@@ -565,7 +568,7 @@ func openRegular(filename string) (*os.File, error) {
 	return f, nil
 }
 
-func (b *Beat) loadDashboards(force bool) error {
+func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
 	if setup || force {
 		// -setup implies dashboards.enabled=true
 		if b.Config.Dashboards == nil {
@@ -583,7 +586,7 @@ func (b *Beat) loadDashboards(force bool) error {
 		if b.Config.Output.Name() == "elasticsearch" {
 			esConfig = b.Config.Output.Config()
 		}
-		err := dashboards.ImportDashboards(b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""),
+		err := dashboards.ImportDashboards(ctx, b.Info.Beat, b.Info.Hostname, paths.Resolve(paths.Home, ""),
 			b.Config.Kibana, esConfig, b.Config.Dashboards, nil)
 		if err != nil {
 			return fmt.Errorf("Error importing Kibana dashboards: %v", err)

libbeat/dashboards/config.go

@@ -1,5 +1,7 @@
 package dashboards
 
+import "time"
+
 type Config struct {
 	Enabled        bool   `config:"enabled"`
 	KibanaIndex    string `config:"kibana_index"`
@@ -11,11 +13,23 @@ type Config struct {
 	OnlyDashboards bool   `config:"only_dashboards"`
 	OnlyIndex      bool   `config:"only_index"`
 	AlwaysKibana   bool   `config:"always_kibana"`
+	Retry          *Retry `config:"retry"`
+}
+
+type Retry struct {
+	Enabled  bool          `config:"enabled"`
+	Interval time.Duration `config:"interval"`
+	Maximum  uint          `config:"maximum"`
 }
 
 var defaultConfig = Config{
 	KibanaIndex:  ".kibana",
 	AlwaysKibana: false,
+	Retry: &Retry{
+		Enabled:  false,
+		Interval: time.Second,
+		Maximum:  0,
+	},
 }
 var (
 	defaultDirectory = "kibana"

libbeat/dashboards/dashboards.go

@@ -1,6 +1,7 @@
 package dashboards
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -24,6 +25,7 @@ const (
 // via the kibana dashboard loader plugin. For older versions of the Elastic Stack
 // we write the dashboards directly into the .kibana index.
 func ImportDashboards(
+	ctx context.Context,
 	beatName, hostname, homePath string,
 	kibanaConfig, esConfig, dashboardsConfig *common.Config,
 	msgOutputter MessageOutputter,
@@ -102,16 +104,16 @@ func ImportDashboards(
 	case importViaES:
 		return ImportDashboardsViaElasticsearch(esLoader)
 	case importViaKibana:
-		return setupAndImportDashboardsViaKibana(hostname, kibanaConfig, &dashConfig, msgOutputter)
+		return setupAndImportDashboardsViaKibana(ctx, hostname, kibanaConfig, &dashConfig, msgOutputter)
 	default:
 		return errors.New("Elasticsearch or Kibana configuration missing for loading dashboards.")
 	}
 }
 
-func setupAndImportDashboardsViaKibana(hostname string, kibanaConfig *common.Config,
+func setupAndImportDashboardsViaKibana(ctx context.Context, hostname string, kibanaConfig *common.Config,
 	dashboardsConfig *Config, msgOutputter MessageOutputter) error {
 
-	kibanaLoader, err := NewKibanaLoader(kibanaConfig, dashboardsConfig, hostname, msgOutputter)
+	kibanaLoader, err := NewKibanaLoader(ctx, kibanaConfig, dashboardsConfig, hostname, msgOutputter)
 	if err != nil {
 		return fmt.Errorf("fail to create the Kibana loader: %v", err)
 	}

libbeat/dashboards/kibana_loader.go

@@ -1,10 +1,12 @@
 package dashboards
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/url"
+	"time"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
@@ -21,13 +23,13 @@ type KibanaLoader struct {
 	msgOutputter MessageOutputter
 }
 
-func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) {
+func NewKibanaLoader(ctx context.Context, cfg *common.Config, dashboardsConfig *Config, hostname string, msgOutputter MessageOutputter) (*KibanaLoader, error) {
 
 	if cfg == nil || !cfg.Enabled() {
 		return nil, fmt.Errorf("Kibana is not configured or enabled")
 	}
 
-	client, err := kibana.NewKibanaClient(cfg)
+	client, err := getKibanaClient(ctx, cfg, dashboardsConfig.Retry, 0)
 	if err != nil {
 		return nil, fmt.Errorf("Error creating Kibana client: %v", err)
 	}
@@ -45,6 +47,22 @@ func NewKibanaLoader(cfg *common.Config, dashboardsConfig *Config, hostname stri
 	return &loader, nil
 }
 
+func getKibanaClient(ctx context.Context, cfg *common.Config, retryCfg *Retry, retryAttempt uint) (*kibana.Client, error) {
+	client, err := kibana.NewKibanaClient(cfg)
+	if err != nil {
+		if retryCfg.Enabled && (retryCfg.Maximum == 0 || retryCfg.Maximum > retryAttempt) {
+			select {
+			case <-ctx.Done():
+				return nil, err
+			case <-time.After(retryCfg.Interval):
+				return getKibanaClient(ctx, cfg, retryCfg, retryAttempt+1)
+			}
+		}
+		return nil, fmt.Errorf("Error creating Kibana client: %v", err)
+	}
+	return client, nil
+}
+
 func (loader KibanaLoader) ImportIndex(file string) error {
 	params := url.Values{}
 	params.Set("force", "true") //overwrite the existing dashboards

libbeat/docs/dashboardsconfig.asciidoc

@@ -99,3 +99,20 @@ NOTE: This setting only works for Kibana 6.0 and newer.
 
 Force loading of dashboards using the Kibana API without querying Elasticsearch for the version
 The default is `false`.
+
+[float]
+==== `setup.dashboards.retry.enabled`
+
+If this option is set to true, and Kibana is not reachable at the time when dashboards are loaded,
+ {beatname_uc} will retry to reconnect to Kibana instead of exiting with an error. Disabled by default.
+
+[float]
+==== `setup.dashboards.retry.interval`
+
+Duration interval between Kibana connection retries. Defaults to 1 second.
+
+[float]
+==== `setup.dashboards.retry.maximum`
+
+Maximum number of retries before exiting with an error. Set to 0 for unlimited retrying.
+Default is unlimited.

libbeat/service/service.go

@@ -1,6 +1,7 @@
 package service
 
 import (
+	"context"
 	"expvar"
 	"flag"
 	"fmt"
@@ -22,7 +23,7 @@ import (
 // HandleSignals manages OS signals that ask the service/daemon to stop.
 // The stopFunction should break the loop in the Beat so that
 // the service shut downs gracefully.
-func HandleSignals(stopFunction func()) {
+func HandleSignals(stopFunction func(), cancel context.CancelFunc) {
 	var callback sync.Once
 
 	// On ^C or SIGTERM, gracefully stop the sniffer
@@ -31,6 +32,7 @@ func HandleSignals(stopFunction func()) {
 	go func() {
 		<-sigc
 		logp.Debug("service", "Received sigterm/sigint, stopping")
+		cancel()
 		callback.Do(stopFunction)
 	}()
 

metricbeat/metricbeat.reference.yml

@@ -1147,6 +1147,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

packetbeat/packetbeat.reference.yml

@@ -1107,6 +1107,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch

winlogbeat/winlogbeat.reference.yml

@@ -668,6 +668,17 @@ output.elasticsearch:
 # how to install the dashboards by first querying Elasticsearch.
 #setup.dashboards.always_kibana: false
 
+# If true and Kibana is not reachable at the time when dashboards are loaded,
+# it will retry to reconnect to Kibana instead of exiting with an error.
+#setup.dashboards.retry.enabled: false
+
+# Duration interval between Kibana connection retries.
+#setup.dashboards.retry.interval: 1s
+
+# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
+#setup.dashboards.retry.maximum: 0
+
+
 #============================== Template =====================================
 
 # A template is used to set the mapping in Elasticsearch