From 579a791349429bc38ec1a7df36f194176b830917 Mon Sep 17 00:00:00 2001
From: Zach <44234405+zjhuber@users.noreply.github.com>
Date: Mon, 25 Oct 2021 07:22:22 -0700
Subject: [PATCH] feat(dot/telemetry): Added connection retry (#1904)

---
 dot/telemetry/telemetry.go | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/dot/telemetry/telemetry.go b/dot/telemetry/telemetry.go
index 2b9fd4081f..af08860190 100644
--- a/dot/telemetry/telemetry.go
+++ b/dot/telemetry/telemetry.go
@@ -43,6 +43,8 @@ type Handler struct {
 	connections        []*telemetryConnection
 	log                log.Logger
 	sendMessageTimeout time.Duration
+	maxRetries         int
+	retryDelay         time.Duration
 }
 
 // Instance interface that telemetry handler instance needs to implement
@@ -61,7 +63,11 @@ var (
 	initilised sync.Once
 )
 
-const defaultMessageTimeout = time.Second
+const (
+	defaultMessageTimeout = time.Second
+	defaultMaxRetries     = 5
+	defaultRetryDelay     = time.Second * 15
+)
 
 // GetInstance singleton pattern to for accessing TelemetryHandler
 func GetInstance() Instance {
@@ -72,6 +78,8 @@ func GetInstance() Instance {
 					msg:                make(chan Message, 256),
 					log:                log.New("pkg", "telemetry"),
 					sendMessageTimeout: defaultMessageTimeout,
+					maxRetries:         defaultMaxRetries,
+					retryDelay:         defaultRetryDelay,
 				}
 				go handlerInstance.startListening()
 			})
@@ -94,17 +102,19 @@ func (h *Handler) Initialise(e bool) {
 // AddConnections adds the given telemetry endpoint as listeners that will receive telemetry data
 func (h *Handler) AddConnections(conns []*genesis.TelemetryEndpoint) {
 	for _, v := range conns {
-		c, _, err := websocket.DefaultDialer.Dial(v.Endpoint, nil)
-		if err != nil {
-			// TODO: try reconnecting if there is an error connecting (#1862)
-			h.log.Debug("issue adding telemetry connection", "error", err)
-			continue
-		}
-		tConn := &telemetryConnection{
-			wsconn:    c,
-			verbosity: v.Verbosity,
+		for connAttempts := 0; connAttempts < h.maxRetries; connAttempts++ {
+			c, _, err := websocket.DefaultDialer.Dial(v.Endpoint, nil)
+			if err != nil {
+				h.log.Debug("issue adding telemetry connection", "error", err)
+				time.Sleep(h.retryDelay)
+				continue
+			}
+			h.connections = append(h.connections, &telemetryConnection{
+				wsconn:    c,
+				verbosity: v.Verbosity,
+			})
+			break
 		}
-		h.connections = append(h.connections, tConn)
 	}
 }